rocketjob 3.5.2 → 4.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (75) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +63 -1
  3. data/bin/rocketjob +1 -0
  4. data/bin/rocketjob_batch_perf +11 -0
  5. data/lib/rocket_job/batch.rb +32 -0
  6. data/lib/rocket_job/batch/callbacks.rb +40 -0
  7. data/lib/rocket_job/batch/io.rb +154 -0
  8. data/lib/rocket_job/batch/logger.rb +57 -0
  9. data/lib/rocket_job/batch/lower_priority.rb +54 -0
  10. data/lib/rocket_job/batch/model.rb +157 -0
  11. data/lib/rocket_job/batch/performance.rb +99 -0
  12. data/lib/rocket_job/batch/result.rb +8 -0
  13. data/lib/rocket_job/batch/results.rb +9 -0
  14. data/lib/rocket_job/batch/state_machine.rb +102 -0
  15. data/lib/rocket_job/batch/statistics.rb +88 -0
  16. data/lib/rocket_job/batch/tabular.rb +56 -0
  17. data/lib/rocket_job/batch/tabular/input.rb +123 -0
  18. data/lib/rocket_job/batch/tabular/output.rb +59 -0
  19. data/lib/rocket_job/batch/throttle.rb +91 -0
  20. data/lib/rocket_job/batch/throttle_running_slices.rb +53 -0
  21. data/lib/rocket_job/batch/worker.rb +288 -0
  22. data/lib/rocket_job/cli.rb +29 -7
  23. data/lib/rocket_job/config.rb +1 -1
  24. data/lib/rocket_job/extensions/mongoid/clients/options.rb +37 -0
  25. data/lib/rocket_job/extensions/mongoid/contextual/mongo.rb +17 -0
  26. data/lib/rocket_job/extensions/mongoid/factory.rb +4 -4
  27. data/lib/rocket_job/extensions/mongoid_5/clients/options.rb +38 -0
  28. data/lib/rocket_job/extensions/mongoid_5/contextual/mongo.rb +64 -0
  29. data/lib/rocket_job/extensions/mongoid_5/factory.rb +13 -0
  30. data/lib/rocket_job/jobs/on_demand_batch_job.rb +127 -0
  31. data/lib/rocket_job/jobs/performance_job.rb +18 -0
  32. data/lib/rocket_job/jobs/upload_file_job.rb +2 -5
  33. data/lib/rocket_job/plugins/document.rb +2 -8
  34. data/lib/rocket_job/plugins/job/persistence.rb +6 -4
  35. data/lib/rocket_job/plugins/job/throttle.rb +3 -6
  36. data/lib/rocket_job/plugins/job/worker.rb +2 -2
  37. data/lib/rocket_job/server.rb +14 -3
  38. data/lib/rocket_job/sliced/input.rb +336 -0
  39. data/lib/rocket_job/sliced/output.rb +99 -0
  40. data/lib/rocket_job/sliced/slice.rb +166 -0
  41. data/lib/rocket_job/sliced/slices.rb +166 -0
  42. data/lib/rocket_job/sliced/writer/input.rb +60 -0
  43. data/lib/rocket_job/sliced/writer/output.rb +82 -0
  44. data/lib/rocket_job/version.rb +1 -1
  45. data/lib/rocket_job/worker.rb +2 -2
  46. data/lib/rocketjob.rb +28 -0
  47. metadata +51 -62
  48. data/test/config/database.yml +0 -5
  49. data/test/config/mongoid.yml +0 -88
  50. data/test/config_test.rb +0 -10
  51. data/test/dirmon_entry_test.rb +0 -313
  52. data/test/dirmon_job_test.rb +0 -216
  53. data/test/files/text.txt +0 -3
  54. data/test/job_test.rb +0 -71
  55. data/test/jobs/housekeeping_job_test.rb +0 -102
  56. data/test/jobs/on_demand_job_test.rb +0 -59
  57. data/test/jobs/upload_file_job_test.rb +0 -107
  58. data/test/plugins/cron_test.rb +0 -166
  59. data/test/plugins/job/callbacks_test.rb +0 -166
  60. data/test/plugins/job/defaults_test.rb +0 -53
  61. data/test/plugins/job/logger_test.rb +0 -56
  62. data/test/plugins/job/model_test.rb +0 -94
  63. data/test/plugins/job/persistence_test.rb +0 -94
  64. data/test/plugins/job/state_machine_test.rb +0 -116
  65. data/test/plugins/job/throttle_test.rb +0 -111
  66. data/test/plugins/job/worker_test.rb +0 -199
  67. data/test/plugins/processing_window_test.rb +0 -109
  68. data/test/plugins/restart_test.rb +0 -193
  69. data/test/plugins/retry_test.rb +0 -88
  70. data/test/plugins/singleton_test.rb +0 -92
  71. data/test/plugins/state_machine_event_callbacks_test.rb +0 -102
  72. data/test/plugins/state_machine_test.rb +0 -67
  73. data/test/plugins/transaction_test.rb +0 -84
  74. data/test/test_db.sqlite3 +0 -0
  75. data/test/test_helper.rb +0 -17
@@ -0,0 +1,82 @@
1
+ module RocketJob
2
+ module Sliced
3
+ module Writer
4
+ # Internal class for writing categorized results into output slices
5
+ class Output
6
+ attr_reader :job, :categorized_records
7
+ attr_accessor :input_slice
8
+
9
+ # Collect output results and write to output collections
10
+ # iff job is collecting output
11
+ # Notes:
12
+ # Nothing is saved if an exception is raised inside the block
13
+ def self.collect(job, input_slice = nil)
14
+ if job.collect_output?
15
+ writer = new(job, input_slice)
16
+ yield(writer)
17
+ writer.close
18
+ else
19
+ writer = NullWriter.new(job, input_slice)
20
+ yield(writer)
21
+ end
22
+ end
23
+
24
+ def initialize(job, input_slice = nil)
25
+ @job = job
26
+ @input_slice = input_slice
27
+ @categorized_records = {}
28
+ end
29
+
30
+ # Writes the supplied result, RocketJob::Batch::Result or RocketJob::Batch::Results
31
+ # to the relevant collections
32
+ def <<(result)
33
+ if result.is_a?(RocketJob::Batch::Results)
34
+ result.each { |single| extract_categorized_result(single) }
35
+ else
36
+ extract_categorized_result(result)
37
+ end
38
+ end
39
+
40
+ # Write categorized results to their relevant collections
41
+ def close
42
+ categorized_records.each_pair do |category, results|
43
+ job.output(category).insert(results, input_slice)
44
+ end
45
+ end
46
+
47
+ private
48
+
49
+ # Stores the categorized result from one result
50
+ def extract_categorized_result(result)
51
+ category = :main
52
+ value = result
53
+ if result.is_a?(RocketJob::Batch::Result)
54
+ category = result.category
55
+ value = result.value
56
+ raise(ArgumentError, "Invalid RocketJob Output Category: #{category}") if job.output_categories.exclude?(category)
57
+ end
58
+ (categorized_records[category] ||= []) << value unless value.nil? && !job.collect_nil_output?
59
+ end
60
+ end
61
+
62
+ class NullWriter
63
+ attr_reader :job, :categorized_records
64
+ attr_accessor :input_slice
65
+
66
+ def initialize(job, input_slice = nil)
67
+ @job = job
68
+ @input_slice = input_slice
69
+ @categorized_records = {}
70
+ end
71
+
72
+ def <<(_)
73
+ # noop
74
+ end
75
+
76
+ def close
77
+ # noop
78
+ end
79
+ end
80
+ end
81
+ end
82
+ end
@@ -1,3 +1,3 @@
1
1
  module RocketJob
2
- VERSION = '3.5.2'.freeze
2
+ VERSION = '4.0.0'.freeze
3
3
  end
@@ -47,7 +47,7 @@ module RocketJob
47
47
  @name = "#{server_name}:#{id}"
48
48
  @re_check_seconds = (re_check_seconds || 60).to_f
49
49
  @re_check_start = Time.now
50
- @filter = filter || {}
50
+ @filter = filter.nil? ? {} : filter.dup
51
51
  @current_filter = @filter.dup
52
52
  @thread = Thread.new { run } unless inline
53
53
  end
@@ -121,7 +121,7 @@ module RocketJob
121
121
  return unless (time - @re_check_start) > re_check_seconds
122
122
 
123
123
  @re_check_start = time
124
- self.current_filter = filter.dup
124
+ self.current_filter = filter.dup if current_filter != filter
125
125
  end
126
126
  end
127
127
  end
@@ -1,12 +1,25 @@
1
+ require 'iostreams'
1
2
  require 'semantic_logger'
2
3
  require 'mongoid'
3
4
  require 'rocket_job/extensions/mongo/logging'
4
5
  require 'rocket_job/version'
5
6
  require 'rocket_job/rocket_job'
6
7
 
8
+ # Apply patches to implement `with_collection`
9
+ if Mongoid::VERSION.to_i >= 6
10
+ require 'rocket_job/extensions/mongoid/clients/options'
11
+ require 'rocket_job/extensions/mongoid/contextual/mongo'
12
+ require 'rocket_job/extensions/mongoid/factory'
13
+ else
14
+ require 'rocket_job/extensions/mongoid_5/clients/options'
15
+ require 'rocket_job/extensions/mongoid_5/contextual/mongo'
16
+ require 'rocket_job/extensions/mongoid_5/factory'
17
+ end
18
+
7
19
  # @formatter:off
8
20
  module RocketJob
9
21
  autoload :ActiveWorker, 'rocket_job/active_worker'
22
+ autoload :Batch, 'rocket_job/batch'
10
23
  autoload :CLI, 'rocket_job/cli'
11
24
  autoload :Config, 'rocket_job/config'
12
25
  autoload :DirmonEntry, 'rocket_job/dirmon_entry'
@@ -47,11 +60,26 @@ module RocketJob
47
60
  module Jobs
48
61
  autoload :ActiveJob, 'rocket_job/jobs/active_job'
49
62
  autoload :DirmonJob, 'rocket_job/jobs/dirmon_job'
63
+ autoload :OnDemandBatchJob, 'rocket_job/jobs/on_demand_batch_job'
50
64
  autoload :OnDemandJob, 'rocket_job/jobs/on_demand_job'
51
65
  autoload :HousekeepingJob, 'rocket_job/jobs/housekeeping_job'
66
+ autoload :PerformanceJob, 'rocket_job/jobs/performance_job'
52
67
  autoload :SimpleJob, 'rocket_job/jobs/simple_job'
53
68
  autoload :UploadFileJob, 'rocket_job/jobs/upload_file_job'
54
69
  end
70
+
71
+ module Sliced
72
+ autoload :Input, 'rocket_job/sliced/input'
73
+ autoload :Output, 'rocket_job/sliced/output'
74
+ autoload :Slice, 'rocket_job/sliced/slice'
75
+ autoload :Slices, 'rocket_job/sliced/slices'
76
+ autoload :Store, 'rocket_job/sliced/store'
77
+
78
+ module Writer
79
+ autoload :Input, 'rocket_job/sliced/writer/input'
80
+ autoload :Output, 'rocket_job/sliced/writer/output'
81
+ end
82
+ end
55
83
  end
56
84
 
57
85
  # Add Active Job adapter for Rails
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: rocketjob
3
3
  version: !ruby/object:Gem::Version
4
- version: 3.5.2
4
+ version: 4.0.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Reid Morrison
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2018-05-31 00:00:00.000000000 Z
11
+ date: 2018-11-14 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: aasm
@@ -38,20 +38,34 @@ dependencies:
38
38
  - - "~>"
39
39
  - !ruby/object:Gem::Version
40
40
  version: '1.0'
41
+ - !ruby/object:Gem::Dependency
42
+ name: iostreams
43
+ requirement: !ruby/object:Gem::Requirement
44
+ requirements:
45
+ - - "~>"
46
+ - !ruby/object:Gem::Version
47
+ version: '0.16'
48
+ type: :runtime
49
+ prerelease: false
50
+ version_requirements: !ruby/object:Gem::Requirement
51
+ requirements:
52
+ - - "~>"
53
+ - !ruby/object:Gem::Version
54
+ version: '0.16'
41
55
  - !ruby/object:Gem::Dependency
42
56
  name: mongoid
43
57
  requirement: !ruby/object:Gem::Requirement
44
58
  requirements:
45
59
  - - ">="
46
60
  - !ruby/object:Gem::Version
47
- version: '5.0'
61
+ version: '5.4'
48
62
  type: :runtime
49
63
  prerelease: false
50
64
  version_requirements: !ruby/object:Gem::Requirement
51
65
  requirements:
52
66
  - - ">="
53
67
  - !ruby/object:Gem::Version
54
- version: '5.0'
68
+ version: '5.4'
55
69
  - !ruby/object:Gem::Dependency
56
70
  name: semantic_logger
57
71
  requirement: !ruby/object:Gem::Requirement
@@ -78,13 +92,36 @@ files:
78
92
  - LICENSE.txt
79
93
  - README.md
80
94
  - bin/rocketjob
95
+ - bin/rocketjob_batch_perf
81
96
  - bin/rocketjob_perf
82
97
  - lib/rocket_job/active_worker.rb
98
+ - lib/rocket_job/batch.rb
99
+ - lib/rocket_job/batch/callbacks.rb
100
+ - lib/rocket_job/batch/io.rb
101
+ - lib/rocket_job/batch/logger.rb
102
+ - lib/rocket_job/batch/lower_priority.rb
103
+ - lib/rocket_job/batch/model.rb
104
+ - lib/rocket_job/batch/performance.rb
105
+ - lib/rocket_job/batch/result.rb
106
+ - lib/rocket_job/batch/results.rb
107
+ - lib/rocket_job/batch/state_machine.rb
108
+ - lib/rocket_job/batch/statistics.rb
109
+ - lib/rocket_job/batch/tabular.rb
110
+ - lib/rocket_job/batch/tabular/input.rb
111
+ - lib/rocket_job/batch/tabular/output.rb
112
+ - lib/rocket_job/batch/throttle.rb
113
+ - lib/rocket_job/batch/throttle_running_slices.rb
114
+ - lib/rocket_job/batch/worker.rb
83
115
  - lib/rocket_job/cli.rb
84
116
  - lib/rocket_job/config.rb
85
117
  - lib/rocket_job/dirmon_entry.rb
86
118
  - lib/rocket_job/extensions/mongo/logging.rb
119
+ - lib/rocket_job/extensions/mongoid/clients/options.rb
120
+ - lib/rocket_job/extensions/mongoid/contextual/mongo.rb
87
121
  - lib/rocket_job/extensions/mongoid/factory.rb
122
+ - lib/rocket_job/extensions/mongoid_5/clients/options.rb
123
+ - lib/rocket_job/extensions/mongoid_5/contextual/mongo.rb
124
+ - lib/rocket_job/extensions/mongoid_5/factory.rb
88
125
  - lib/rocket_job/extensions/rocket_job_adapter.rb
89
126
  - lib/rocket_job/heartbeat.rb
90
127
  - lib/rocket_job/job.rb
@@ -92,7 +129,9 @@ files:
92
129
  - lib/rocket_job/jobs/active_job.rb
93
130
  - lib/rocket_job/jobs/dirmon_job.rb
94
131
  - lib/rocket_job/jobs/housekeeping_job.rb
132
+ - lib/rocket_job/jobs/on_demand_batch_job.rb
95
133
  - lib/rocket_job/jobs/on_demand_job.rb
134
+ - lib/rocket_job/jobs/performance_job.rb
96
135
  - lib/rocket_job/jobs/simple_job.rb
97
136
  - lib/rocket_job/jobs/upload_file_job.rb
98
137
  - lib/rocket_job/performance.rb
@@ -116,37 +155,15 @@ files:
116
155
  - lib/rocket_job/plugins/transaction.rb
117
156
  - lib/rocket_job/rocket_job.rb
118
157
  - lib/rocket_job/server.rb
158
+ - lib/rocket_job/sliced/input.rb
159
+ - lib/rocket_job/sliced/output.rb
160
+ - lib/rocket_job/sliced/slice.rb
161
+ - lib/rocket_job/sliced/slices.rb
162
+ - lib/rocket_job/sliced/writer/input.rb
163
+ - lib/rocket_job/sliced/writer/output.rb
119
164
  - lib/rocket_job/version.rb
120
165
  - lib/rocket_job/worker.rb
121
166
  - lib/rocketjob.rb
122
- - test/config/database.yml
123
- - test/config/mongoid.yml
124
- - test/config_test.rb
125
- - test/dirmon_entry_test.rb
126
- - test/dirmon_job_test.rb
127
- - test/files/text.txt
128
- - test/job_test.rb
129
- - test/jobs/housekeeping_job_test.rb
130
- - test/jobs/on_demand_job_test.rb
131
- - test/jobs/upload_file_job_test.rb
132
- - test/plugins/cron_test.rb
133
- - test/plugins/job/callbacks_test.rb
134
- - test/plugins/job/defaults_test.rb
135
- - test/plugins/job/logger_test.rb
136
- - test/plugins/job/model_test.rb
137
- - test/plugins/job/persistence_test.rb
138
- - test/plugins/job/state_machine_test.rb
139
- - test/plugins/job/throttle_test.rb
140
- - test/plugins/job/worker_test.rb
141
- - test/plugins/processing_window_test.rb
142
- - test/plugins/restart_test.rb
143
- - test/plugins/retry_test.rb
144
- - test/plugins/singleton_test.rb
145
- - test/plugins/state_machine_event_callbacks_test.rb
146
- - test/plugins/state_machine_test.rb
147
- - test/plugins/transaction_test.rb
148
- - test/test_db.sqlite3
149
- - test/test_helper.rb
150
167
  homepage: http://rocketjob.io
151
168
  licenses:
152
169
  - Apache-2.0
@@ -167,36 +184,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
167
184
  version: '0'
168
185
  requirements: []
169
186
  rubyforge_project:
170
- rubygems_version: 2.7.6
187
+ rubygems_version: 2.7.7
171
188
  signing_key:
172
189
  specification_version: 4
173
190
  summary: Ruby's missing batch system.
174
- test_files:
175
- - test/config_test.rb
176
- - test/config/mongoid.yml
177
- - test/config/database.yml
178
- - test/test_db.sqlite3
179
- - test/plugins/state_machine_test.rb
180
- - test/plugins/restart_test.rb
181
- - test/plugins/state_machine_event_callbacks_test.rb
182
- - test/plugins/transaction_test.rb
183
- - test/plugins/cron_test.rb
184
- - test/plugins/job/logger_test.rb
185
- - test/plugins/job/state_machine_test.rb
186
- - test/plugins/job/worker_test.rb
187
- - test/plugins/job/throttle_test.rb
188
- - test/plugins/job/defaults_test.rb
189
- - test/plugins/job/persistence_test.rb
190
- - test/plugins/job/callbacks_test.rb
191
- - test/plugins/job/model_test.rb
192
- - test/plugins/singleton_test.rb
193
- - test/plugins/processing_window_test.rb
194
- - test/plugins/retry_test.rb
195
- - test/job_test.rb
196
- - test/dirmon_job_test.rb
197
- - test/dirmon_entry_test.rb
198
- - test/files/text.txt
199
- - test/test_helper.rb
200
- - test/jobs/housekeeping_job_test.rb
201
- - test/jobs/on_demand_job_test.rb
202
- - test/jobs/upload_file_job_test.rb
191
+ test_files: []
@@ -1,5 +0,0 @@
1
- test:
2
- adapter: sqlite3
3
- database: test/test_db.sqlite3
4
- pool: 5
5
- timeout: 5000
@@ -1,88 +0,0 @@
1
- # See: https://docs.mongodb.com/ruby-driver/master/tutorials/5.1.0/mongoid-installation/
2
- client_options: &client_options
3
- read:
4
- mode: :primary
5
- write:
6
- w: 0
7
- max_pool_size: 50
8
- min_pool_size: 10
9
- connect_timeout: 5
10
- socket_timeout: 300
11
- wait_queue_timeout: 5
12
-
13
- mongoid_options: &mongoid_options
14
- # Includes the root model name in json serialization. (default: false)
15
- # include_root_in_json: false
16
-
17
- # Include the _type field in serialization. (default: false)
18
- # include_type_for_serialization: false
19
-
20
- # Preload all models in development, needed when models use
21
- # inheritance. (default: false)
22
- preload_models: true
23
-
24
- # Raise an error when performing a #find and the document is not found.
25
- # (default: true)
26
- # raise_not_found_error: true
27
-
28
- # Raise an error when defining a scope with the same name as an
29
- # existing method. (default: false)
30
- scope_overwrite_exception: true
31
-
32
- # Use Active Support's time zone in conversions. (default: true)
33
- # use_activesupport_time_zone: true
34
-
35
- # Ensure all times are UTC in the app side. (default: false)
36
- use_utc: true
37
-
38
- development:
39
- clients:
40
- default: &default_development
41
- uri: mongodb://127.0.0.1:27017/rocketjob_development
42
- options:
43
- <<: *client_options
44
- write:
45
- w: 0
46
- max_pool_size: 5
47
- min_pool_size: 1
48
- rocketjob:
49
- <<: *default_development
50
- rocketjob_slices:
51
- <<: *default_development
52
- options:
53
- <<: *mongoid_options
54
-
55
- test:
56
- clients:
57
- default: &default_test
58
- uri: mongodb://127.0.0.1:27017/rocketjob_test
59
- options:
60
- <<: *client_options
61
- write:
62
- w: 1
63
- max_pool_size: 5
64
- min_pool_size: 1
65
- rocketjob:
66
- <<: *default_test
67
- rocketjob_slices:
68
- <<: *default_test
69
- options:
70
- <<: *mongoid_options
71
-
72
- production:
73
- clients:
74
- default: &default_production
75
- uri: mongodb://user:secret@server.example.org:27017,server2.example.org:27017/rocketjob_production
76
- options:
77
- <<: *client_options
78
- write:
79
- w: 0
80
- rocketjob:
81
- <<: *default_production
82
- rocketjob_slices:
83
- <<: *default_production
84
- # Optionally Specify a different database or even server to store slices on
85
- # uri: mongodb://user:secret@server3.example.org:27017/slices_production
86
- options:
87
- <<: *mongoid_options
88
-
@@ -1,10 +0,0 @@
1
- require_relative 'test_helper'
2
- class ConfigTest < Minitest::Test
3
- describe RocketJob::Config do
4
- describe '.config' do
5
- it 'support multiple databases' do
6
- assert_equal 'rocketjob_test', RocketJob::Job.collection.database.name
7
- end
8
- end
9
- end
10
- end