rocketjob 5.4.0.beta1 → 6.0.0.rc2

Sign up to get free protection for your applications and to get access to all the features.
Files changed (60) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +19 -5
  3. data/bin/rocketjob_batch_perf +1 -1
  4. data/bin/rocketjob_perf +1 -1
  5. data/lib/rocket_job/batch.rb +3 -0
  6. data/lib/rocket_job/batch/categories.rb +341 -0
  7. data/lib/rocket_job/batch/io.rb +128 -60
  8. data/lib/rocket_job/batch/model.rb +20 -68
  9. data/lib/rocket_job/batch/performance.rb +19 -7
  10. data/lib/rocket_job/batch/statistics.rb +34 -12
  11. data/lib/rocket_job/batch/tabular.rb +2 -0
  12. data/lib/rocket_job/batch/tabular/input.rb +8 -6
  13. data/lib/rocket_job/batch/tabular/output.rb +4 -2
  14. data/lib/rocket_job/batch/throttle_running_workers.rb +8 -17
  15. data/lib/rocket_job/batch/worker.rb +27 -24
  16. data/lib/rocket_job/category/base.rb +78 -0
  17. data/lib/rocket_job/category/input.rb +110 -0
  18. data/lib/rocket_job/category/output.rb +25 -0
  19. data/lib/rocket_job/cli.rb +25 -17
  20. data/lib/rocket_job/dirmon_entry.rb +22 -12
  21. data/lib/rocket_job/event.rb +1 -1
  22. data/lib/rocket_job/extensions/iostreams/path.rb +32 -0
  23. data/lib/rocket_job/extensions/mongoid/contextual/mongo.rb +2 -2
  24. data/lib/rocket_job/extensions/mongoid/factory.rb +4 -12
  25. data/lib/rocket_job/extensions/mongoid/stringified_symbol.rb +50 -0
  26. data/lib/rocket_job/extensions/psych/yaml_tree.rb +8 -0
  27. data/lib/rocket_job/extensions/rocket_job_adapter.rb +2 -2
  28. data/lib/rocket_job/jobs/dirmon_job.rb +2 -2
  29. data/lib/rocket_job/jobs/housekeeping_job.rb +7 -7
  30. data/lib/rocket_job/jobs/on_demand_batch_job.rb +15 -6
  31. data/lib/rocket_job/jobs/on_demand_job.rb +1 -2
  32. data/lib/rocket_job/jobs/performance_job.rb +3 -1
  33. data/lib/rocket_job/jobs/re_encrypt/relational_job.rb +103 -96
  34. data/lib/rocket_job/jobs/upload_file_job.rb +44 -8
  35. data/lib/rocket_job/lookup_collection.rb +69 -0
  36. data/lib/rocket_job/plugins/job/model.rb +25 -50
  37. data/lib/rocket_job/plugins/job/throttle.rb +2 -2
  38. data/lib/rocket_job/plugins/job/throttle_running_jobs.rb +12 -4
  39. data/lib/rocket_job/plugins/job/worker.rb +2 -7
  40. data/lib/rocket_job/plugins/restart.rb +12 -5
  41. data/lib/rocket_job/plugins/state_machine.rb +2 -1
  42. data/lib/rocket_job/plugins/throttle_dependent_jobs.rb +38 -0
  43. data/lib/rocket_job/ractor_worker.rb +42 -0
  44. data/lib/rocket_job/server/model.rb +1 -1
  45. data/lib/rocket_job/sliced.rb +15 -70
  46. data/lib/rocket_job/sliced/bzip2_output_slice.rb +2 -2
  47. data/lib/rocket_job/sliced/input.rb +1 -1
  48. data/lib/rocket_job/sliced/slice.rb +5 -13
  49. data/lib/rocket_job/sliced/slices.rb +14 -2
  50. data/lib/rocket_job/sliced/writer/output.rb +33 -45
  51. data/lib/rocket_job/subscribers/server.rb +1 -1
  52. data/lib/rocket_job/thread_worker.rb +46 -0
  53. data/lib/rocket_job/throttle_definitions.rb +7 -1
  54. data/lib/rocket_job/version.rb +1 -1
  55. data/lib/rocket_job/worker.rb +21 -55
  56. data/lib/rocket_job/worker_pool.rb +5 -7
  57. data/lib/rocketjob.rb +53 -43
  58. metadata +35 -26
  59. data/lib/rocket_job/extensions/mongoid/remove_warnings.rb +0 -12
  60. data/lib/rocket_job/jobs/on_demand_batch_tabular_job.rb +0 -28
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: rocketjob
3
3
  version: !ruby/object:Gem::Version
4
- version: 5.4.0.beta1
4
+ version: 6.0.0.rc2
5
5
  platform: ruby
6
6
  authors:
7
7
  - Reid Morrison
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2020-10-21 00:00:00.000000000 Z
11
+ date: 2021-06-16 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: aasm
@@ -16,14 +16,14 @@ dependencies:
16
16
  requirements:
17
17
  - - ">="
18
18
  - !ruby/object:Gem::Version
19
- version: '4.12'
19
+ version: '5.1'
20
20
  type: :runtime
21
21
  prerelease: false
22
22
  version_requirements: !ruby/object:Gem::Requirement
23
23
  requirements:
24
24
  - - ">="
25
25
  - !ruby/object:Gem::Version
26
- version: '4.12'
26
+ version: '5.1'
27
27
  - !ruby/object:Gem::Dependency
28
28
  name: concurrent-ruby
29
29
  requirement: !ruby/object:Gem::Requirement
@@ -39,75 +39,75 @@ dependencies:
39
39
  - !ruby/object:Gem::Version
40
40
  version: '1.1'
41
41
  - !ruby/object:Gem::Dependency
42
- name: iostreams
42
+ name: fugit
43
43
  requirement: !ruby/object:Gem::Requirement
44
44
  requirements:
45
45
  - - "~>"
46
46
  - !ruby/object:Gem::Version
47
- version: '1.2'
47
+ version: '1.4'
48
48
  type: :runtime
49
49
  prerelease: false
50
50
  version_requirements: !ruby/object:Gem::Requirement
51
51
  requirements:
52
52
  - - "~>"
53
53
  - !ruby/object:Gem::Version
54
- version: '1.2'
54
+ version: '1.4'
55
55
  - !ruby/object:Gem::Dependency
56
- name: mongoid
56
+ name: iostreams
57
57
  requirement: !ruby/object:Gem::Requirement
58
58
  requirements:
59
59
  - - "~>"
60
60
  - !ruby/object:Gem::Version
61
- version: '7.0'
61
+ version: '1.6'
62
62
  type: :runtime
63
63
  prerelease: false
64
64
  version_requirements: !ruby/object:Gem::Requirement
65
65
  requirements:
66
66
  - - "~>"
67
67
  - !ruby/object:Gem::Version
68
- version: '7.0'
68
+ version: '1.6'
69
69
  - !ruby/object:Gem::Dependency
70
- name: semantic_logger
70
+ name: mongoid
71
71
  requirement: !ruby/object:Gem::Requirement
72
72
  requirements:
73
73
  - - "~>"
74
74
  - !ruby/object:Gem::Version
75
- version: '4.1'
75
+ version: '7.1'
76
76
  type: :runtime
77
77
  prerelease: false
78
78
  version_requirements: !ruby/object:Gem::Requirement
79
79
  requirements:
80
80
  - - "~>"
81
81
  - !ruby/object:Gem::Version
82
- version: '4.1'
82
+ version: '7.1'
83
83
  - !ruby/object:Gem::Dependency
84
- name: symmetric-encryption
84
+ name: semantic_logger
85
85
  requirement: !ruby/object:Gem::Requirement
86
86
  requirements:
87
- - - ">="
87
+ - - "~>"
88
88
  - !ruby/object:Gem::Version
89
- version: '4.0'
89
+ version: '4.7'
90
90
  type: :runtime
91
91
  prerelease: false
92
92
  version_requirements: !ruby/object:Gem::Requirement
93
93
  requirements:
94
- - - ">="
94
+ - - "~>"
95
95
  - !ruby/object:Gem::Version
96
- version: '4.0'
96
+ version: '4.7'
97
97
  - !ruby/object:Gem::Dependency
98
- name: fugit
98
+ name: symmetric-encryption
99
99
  requirement: !ruby/object:Gem::Requirement
100
100
  requirements:
101
101
  - - "~>"
102
102
  - !ruby/object:Gem::Version
103
- version: '1.3'
103
+ version: '4.3'
104
104
  type: :runtime
105
105
  prerelease: false
106
106
  version_requirements: !ruby/object:Gem::Requirement
107
107
  requirements:
108
108
  - - "~>"
109
109
  - !ruby/object:Gem::Version
110
- version: '1.3'
110
+ version: '4.3'
111
111
  description:
112
112
  email:
113
113
  executables:
@@ -124,6 +124,7 @@ files:
124
124
  - lib/rocket_job/active_worker.rb
125
125
  - lib/rocket_job/batch.rb
126
126
  - lib/rocket_job/batch/callbacks.rb
127
+ - lib/rocket_job/batch/categories.rb
127
128
  - lib/rocket_job/batch/io.rb
128
129
  - lib/rocket_job/batch/logger.rb
129
130
  - lib/rocket_job/batch/lower_priority.rb
@@ -140,15 +141,20 @@ files:
140
141
  - lib/rocket_job/batch/throttle_running_workers.rb
141
142
  - lib/rocket_job/batch/throttle_windows.rb
142
143
  - lib/rocket_job/batch/worker.rb
144
+ - lib/rocket_job/category/base.rb
145
+ - lib/rocket_job/category/input.rb
146
+ - lib/rocket_job/category/output.rb
143
147
  - lib/rocket_job/cli.rb
144
148
  - lib/rocket_job/config.rb
145
149
  - lib/rocket_job/dirmon_entry.rb
146
150
  - lib/rocket_job/event.rb
151
+ - lib/rocket_job/extensions/iostreams/path.rb
147
152
  - lib/rocket_job/extensions/mongo/logging.rb
148
153
  - lib/rocket_job/extensions/mongoid/clients/options.rb
149
154
  - lib/rocket_job/extensions/mongoid/contextual/mongo.rb
150
155
  - lib/rocket_job/extensions/mongoid/factory.rb
151
- - lib/rocket_job/extensions/mongoid/remove_warnings.rb
156
+ - lib/rocket_job/extensions/mongoid/stringified_symbol.rb
157
+ - lib/rocket_job/extensions/psych/yaml_tree.rb
152
158
  - lib/rocket_job/extensions/rocket_job_adapter.rb
153
159
  - lib/rocket_job/heartbeat.rb
154
160
  - lib/rocket_job/job.rb
@@ -158,12 +164,12 @@ files:
158
164
  - lib/rocket_job/jobs/dirmon_job.rb
159
165
  - lib/rocket_job/jobs/housekeeping_job.rb
160
166
  - lib/rocket_job/jobs/on_demand_batch_job.rb
161
- - lib/rocket_job/jobs/on_demand_batch_tabular_job.rb
162
167
  - lib/rocket_job/jobs/on_demand_job.rb
163
168
  - lib/rocket_job/jobs/performance_job.rb
164
169
  - lib/rocket_job/jobs/re_encrypt/relational_job.rb
165
170
  - lib/rocket_job/jobs/simple_job.rb
166
171
  - lib/rocket_job/jobs/upload_file_job.rb
172
+ - lib/rocket_job/lookup_collection.rb
167
173
  - lib/rocket_job/performance.rb
168
174
  - lib/rocket_job/plugins/cron.rb
169
175
  - lib/rocket_job/plugins/document.rb
@@ -180,7 +186,9 @@ files:
180
186
  - lib/rocket_job/plugins/retry.rb
181
187
  - lib/rocket_job/plugins/singleton.rb
182
188
  - lib/rocket_job/plugins/state_machine.rb
189
+ - lib/rocket_job/plugins/throttle_dependent_jobs.rb
183
190
  - lib/rocket_job/plugins/transaction.rb
191
+ - lib/rocket_job/ractor_worker.rb
184
192
  - lib/rocket_job/railtie.rb
185
193
  - lib/rocket_job/rocket_job.rb
186
194
  - lib/rocket_job/server.rb
@@ -202,13 +210,14 @@ files:
202
210
  - lib/rocket_job/subscribers/worker.rb
203
211
  - lib/rocket_job/supervisor.rb
204
212
  - lib/rocket_job/supervisor/shutdown.rb
213
+ - lib/rocket_job/thread_worker.rb
205
214
  - lib/rocket_job/throttle_definition.rb
206
215
  - lib/rocket_job/throttle_definitions.rb
207
216
  - lib/rocket_job/version.rb
208
217
  - lib/rocket_job/worker.rb
209
218
  - lib/rocket_job/worker_pool.rb
210
219
  - lib/rocketjob.rb
211
- homepage: http://rocketjob.io
220
+ homepage: https://rocketjob.io
212
221
  licenses:
213
222
  - Apache-2.0
214
223
  metadata: {}
@@ -220,14 +229,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
220
229
  requirements:
221
230
  - - ">="
222
231
  - !ruby/object:Gem::Version
223
- version: '2.3'
232
+ version: '2.5'
224
233
  required_rubygems_version: !ruby/object:Gem::Requirement
225
234
  requirements:
226
235
  - - ">"
227
236
  - !ruby/object:Gem::Version
228
237
  version: 1.3.1
229
238
  requirements: []
230
- rubygems_version: 3.0.8
239
+ rubygems_version: 3.2.15
231
240
  signing_key:
232
241
  specification_version: 4
233
242
  summary: Ruby's missing batch processing system.
@@ -1,12 +0,0 @@
1
- require "mongoid/fields/validators/macro"
2
- require "semantic_logger"
3
- module RocketJob
4
- module RemoveMongoidWarnings
5
- # Remove annoying warnings about Symbols type being deprecated.
6
- def validate_options(*params)
7
- SemanticLogger.silence(:error) { super(*params) }
8
- end
9
- end
10
- end
11
-
12
- ::Mongoid::Fields::Validators::Macro.extend(RocketJob::RemoveMongoidWarnings)
@@ -1,28 +0,0 @@
1
- # Job to dynamically perform ruby code on demand as a Batch,
2
- # with input and/or output from CSV/JSON or other format supported by Tabular.
3
- #
4
- # Nodes:
5
- # - Need to specify `destroy_on_complete: false` to collect output from this job.
6
- # - `after_code` can be used to automatically download the output of this job to a file on completion.
7
- #
8
- # Example: Iterate over all rows in a table:
9
- # code = <<-CODE
10
- # if user = User.find(row)
11
- # user.cleanse_attributes!
12
- # user.save(validate: false)
13
- # end
14
- # CODE
15
- # job = RocketJob::Jobs::OnDemandBatchTabularJob.new(code: code, description: 'cleanse users', destroy_on_complete: false)
16
- # job.upload("users.csv")
17
- # job.save!
18
- #
19
- # On completion export the output:
20
- # job.download("output.csv")
21
- module RocketJob
22
- module Jobs
23
- class OnDemandBatchTabularJob < OnDemandBatchJob
24
- include RocketJob::Batch::Tabular::Input
25
- include RocketJob::Batch::Tabular::Output
26
- end
27
- end
28
- end