rocketjob 7.0.0 → 7.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +1 -1
- data/lib/rocket_job/batch/categories.rb +161 -40
- data/lib/rocket_job/batch/io.rb +239 -133
- data/lib/rocket_job/category/base.rb +176 -1
- data/lib/rocket_job/category/input.rb +357 -41
- data/lib/rocket_job/category/input_schema_header.rb +90 -0
- data/lib/rocket_job/category/input_schema_proposal.rb +221 -0
- data/lib/rocket_job/category/input_text_lines.rb +37 -0
- data/lib/rocket_job/category/output.rb +39 -0
- data/lib/rocket_job/deprecation.rb +61 -0
- data/lib/rocket_job/dirmon_entry.rb +126 -6
- data/lib/rocket_job/extensions/iostreams/path.rb +24 -0
- data/lib/rocket_job/jobs/copy_file_job.rb +114 -20
- data/lib/rocket_job/jobs/dirmon_job.rb +7 -4
- data/lib/rocket_job/jobs/on_demand_batch_job.rb +118 -4
- data/lib/rocket_job/jobs/upload_file_job.rb +57 -3
- data/lib/rocket_job/plugins/job/model.rb +55 -0
- data/lib/rocket_job/plugins/retry.rb +35 -34
- data/lib/rocket_job/rocket_job.rb +33 -0
- data/lib/rocket_job/schema_proposal.rb +232 -0
- data/lib/rocket_job/sliced/input.rb +149 -5
- data/lib/rocket_job/sliced/slice.rb +4 -0
- data/lib/rocket_job/sliced/slices.rb +1 -1
- data/lib/rocket_job/sliced/writer/input.rb +13 -6
- data/lib/rocket_job/version.rb +1 -1
- data/lib/rocketjob.rb +5 -0
- metadata +9 -4
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 60354ecdc42ea57a95256cfb2e1c239ced4449fbf5acd99d6707c7d2948fb06d
|
|
4
|
+
data.tar.gz: e99ef1bddca30c09cf7496c4cdca33b5360a56542230b076541e89ce50fb7aa6
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 529c1e43856e754b4204a3b2f6a24f1b0bcda0e90e19ee980637372d420c97b8ee97213d7d1564e5d5c5f1889ca831faed4d6e765f77b09df81bcaab92f83598
|
|
7
|
+
data.tar.gz: 2e1d359ba192068dad4532b491f796d20352c2263d32572f04b6f684e5b328aff1f1fb413566d3adadc67f92528bacd3143daee1ced7cf90c065743f61751c2f
|
data/README.md
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
# Rocket Job
|
|
2
|
-
[](https://rubygems.org/gems/rocketjob) [](https://rubygems.org/gems/rocketjob) [](http://opensource.org/licenses/Apache-2.0) 
|
|
2
|
+
[](https://rubygems.org/gems/rocketjob) [](https://rubygems.org/gems/rocketjob) [](https://github.com/reidmorrison/rocketjob/actions/workflows/ci.yml) [](http://opensource.org/licenses/Apache-2.0) 
|
|
3
3
|
|
|
4
4
|
**Process millions of records across thousands of workers.**
|
|
5
5
|
|
|
@@ -26,26 +26,24 @@ module RocketJob
|
|
|
26
26
|
end
|
|
27
27
|
|
|
28
28
|
module ClassMethods
|
|
29
|
-
# Define a new input category
|
|
29
|
+
# Define a new input category, or replace the input category with the same name, and return it.
|
|
30
|
+
# A subclass that defines a category does not change the categories of its parent class.
|
|
30
31
|
# @see RocketJob::Category::Input
|
|
31
32
|
def input_category(**args)
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
rocketjob_categories_set(category, defined_input_categories)
|
|
37
|
-
end
|
|
33
|
+
RocketJob::Category::Input.warn_deprecated_options(args)
|
|
34
|
+
category = RocketJob::Category::Input.new(**args)
|
|
35
|
+
self.defined_input_categories = rocketjob_categories_with(defined_input_categories, category)
|
|
36
|
+
category
|
|
38
37
|
end
|
|
39
38
|
|
|
40
|
-
# Define a new output category
|
|
39
|
+
# Define a new output category, or replace the output category with the same name, and return it.
|
|
40
|
+
# A subclass that defines a category does not change the categories of its parent class.
|
|
41
41
|
# @see RocketJob::Category::Output
|
|
42
42
|
def output_category(**args)
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
rocketjob_categories_set(category, defined_output_categories)
|
|
48
|
-
end
|
|
43
|
+
RocketJob::Category::Output.warn_deprecated_options(args)
|
|
44
|
+
category = RocketJob::Category::Output.new(**args)
|
|
45
|
+
self.defined_output_categories = rocketjob_categories_with(defined_output_categories, category)
|
|
46
|
+
category
|
|
49
47
|
end
|
|
50
48
|
|
|
51
49
|
# Builds this job instance from the supplied properties hash that may contain input and output categories.
|
|
@@ -62,12 +60,111 @@ module RocketJob
|
|
|
62
60
|
job
|
|
63
61
|
end
|
|
64
62
|
|
|
63
|
+
# Returns [Array<RocketJob::SchemaProposal>] the schemas proposed to replace the deprecated options of the
|
|
64
|
+
# categories that this job class defines, rather than inherits, see RocketJob::Category::Input#schema_proposal.
|
|
65
|
+
# A class holds the same category as its parent class for each one that it inherits, see
|
|
66
|
+
# #rocketjob_categories_with, and a new one for each that it defines or redefines.
|
|
67
|
+
def schema_proposals
|
|
68
|
+
inherited = superclass.respond_to?(:defined_input_categories) ? rocketjob_categories_of(superclass) : []
|
|
69
|
+
rocketjob_categories_of(self).filter_map do |category|
|
|
70
|
+
next if inherited.any? { |parent| parent.equal?(category) }
|
|
71
|
+
|
|
72
|
+
category.schema_proposal("#{name}, #{rocketjob_category_label(category)}")
|
|
73
|
+
end
|
|
74
|
+
end
|
|
75
|
+
|
|
76
|
+
# Returns [true] since a saved job of this class can hold deprecated category options, see
|
|
77
|
+
# RocketJob::Plugins::Job::Model::ClassMethods#proposes_schemas?.
|
|
78
|
+
def proposes_schemas?
|
|
79
|
+
true
|
|
80
|
+
end
|
|
81
|
+
|
|
82
|
+
# Returns [Array<RocketJob::SchemaProposal>] the schemas proposed to replace the deprecated options in the
|
|
83
|
+
# supplied properties of a job of this class, such as those that a Dirmon entry or an UploadFileJob holds, at
|
|
84
|
+
# the supplied place. Each is proposed for the category that the properties build, with the job class's own
|
|
85
|
+
# settings, or says that a job of this class has no category of that name, for which #from_properties raises.
|
|
86
|
+
def property_schema_proposals(properties, where)
|
|
87
|
+
properties = properties.to_h.stringify_keys
|
|
88
|
+
{"input_categories" => RocketJob::Category::Input, "output_categories" => RocketJob::Category::Output}.flat_map do |key, klass|
|
|
89
|
+
Array(properties[key]).filter_map do |category_properties|
|
|
90
|
+
next unless category_properties.is_a?(Hash)
|
|
91
|
+
|
|
92
|
+
category_properties = category_properties.stringify_keys
|
|
93
|
+
next if klass.deprecated_options(category_properties).empty?
|
|
94
|
+
|
|
95
|
+
category = klass.new(name: (category_properties["name"] || :main).to_sym)
|
|
96
|
+
label = "#{where}, #{rocketjob_category_label(category)}, in its properties"
|
|
97
|
+
built = rocketjob_category_from(category, category_properties)
|
|
98
|
+
next built.schema_proposal(label) if built
|
|
99
|
+
|
|
100
|
+
RocketJob::SchemaProposal.new(
|
|
101
|
+
where: label,
|
|
102
|
+
error: "#{name} has no #{rocketjob_category_label(category)}, so a job cannot be built from these " \
|
|
103
|
+
"properties. Remove them, or define that category in the job class."
|
|
104
|
+
)
|
|
105
|
+
end
|
|
106
|
+
end
|
|
107
|
+
end
|
|
108
|
+
|
|
109
|
+
# Returns [String] how a category is named in a proposal, such as "input category main".
|
|
110
|
+
def rocketjob_category_label(category)
|
|
111
|
+
"#{category.class.name.demodulize.downcase} category #{category.name}"
|
|
112
|
+
end
|
|
113
|
+
|
|
114
|
+
# Returns [RocketJob::Category::Input|RocketJob::Category::Output] the category of the supplied class that a job
|
|
115
|
+
# of this class starts with, by the name of the supplied category, see #rocketjob_categories_assign, or nil when
|
|
116
|
+
# it starts without one.
|
|
117
|
+
def rocketjob_defined_category(category)
|
|
118
|
+
defined = category.is_a?(RocketJob::Category::Input) ? rocketjob_initial_input_categories : defined_output_categories
|
|
119
|
+
defined&.find { |candidate| candidate.name == category.name }
|
|
120
|
+
end
|
|
121
|
+
|
|
122
|
+
# Returns [Array<RocketJob::Category::Input>] the input categories that a job of this class starts with: those
|
|
123
|
+
# that it defines, or the main one when it defines none. Copy them before changing them.
|
|
124
|
+
def rocketjob_initial_input_categories
|
|
125
|
+
defined_input_categories || [RocketJob::Category::Input.new]
|
|
126
|
+
end
|
|
127
|
+
|
|
128
|
+
# Returns [Hash] the supplied attributes or properties of this job class to show to people, see
|
|
129
|
+
# RocketJob::Plugins::Job::Model.display_properties, with each of its input and output categories also
|
|
130
|
+
# showing its file name without any credentials.
|
|
131
|
+
def display_properties(properties)
|
|
132
|
+
super.to_h do |key, value|
|
|
133
|
+
category_class =
|
|
134
|
+
case key.to_s
|
|
135
|
+
when "input_categories" then RocketJob::Category::Input
|
|
136
|
+
when "output_categories" then RocketJob::Category::Output
|
|
137
|
+
end
|
|
138
|
+
next [key, value] unless category_class && value.is_a?(Array)
|
|
139
|
+
|
|
140
|
+
[key, value.map { |category| category.is_a?(Hash) ? category_class.display_properties(category) : category }]
|
|
141
|
+
end
|
|
142
|
+
end
|
|
143
|
+
|
|
65
144
|
private
|
|
66
145
|
|
|
67
|
-
def
|
|
68
|
-
|
|
146
|
+
def rocketjob_categories_of(klass)
|
|
147
|
+
[*klass.defined_input_categories, *klass.defined_output_categories]
|
|
148
|
+
end
|
|
149
|
+
|
|
150
|
+
# Returns the category that the supplied properties build for a job of this class, from its own category of
|
|
151
|
+
# the class and name of the supplied one, see #merge_input_categories, without reporting a deprecation warning,
|
|
152
|
+
# or nil when a job of this class has no such category.
|
|
153
|
+
def rocketjob_category_from(category, properties)
|
|
154
|
+
built = rocketjob_defined_category(category)&.deep_dup
|
|
155
|
+
properties.each_pair { |key, value| built.public_send(:"#{key}=", value) } if built
|
|
156
|
+
built
|
|
157
|
+
end
|
|
158
|
+
|
|
159
|
+
# Returns [Array] a copy of the supplied categories, which can be nil, with the supplied category in place of
|
|
160
|
+
# the one with the same name, or after them. The supplied categories are not changed, since a class shares
|
|
161
|
+
# them with its subclasses. Only the list is copied, not the categories in it, so that .schema_proposals can
|
|
162
|
+
# tell the categories that a class inherits from those that it defines.
|
|
163
|
+
def rocketjob_categories_with(categories, category)
|
|
164
|
+
categories = categories ? categories.dup : []
|
|
165
|
+
index = categories.find_index { |cat| cat.name == category.name }
|
|
69
166
|
index ? categories[index] = category : categories << category
|
|
70
|
-
|
|
167
|
+
categories
|
|
71
168
|
end
|
|
72
169
|
end
|
|
73
170
|
|
|
@@ -120,38 +217,71 @@ module RocketJob
|
|
|
120
217
|
false
|
|
121
218
|
end
|
|
122
219
|
|
|
220
|
+
# Returns [Array<RocketJob::SchemaProposal>] the schemas proposed to replace the deprecated options of this saved
|
|
221
|
+
# job's categories, see RocketJob::Category::Input#schema_proposal. A job that is running, paused or failed has
|
|
222
|
+
# uploaded its input with them, so it is only told to finish first. None for a job that has completed or was
|
|
223
|
+
# aborted, since it cannot run again. What reading its categories raises is answered as one proposal with the
|
|
224
|
+
# error, see RocketJob::SchemaProposal.unreadable.
|
|
225
|
+
def schema_proposals
|
|
226
|
+
where = "#{self.class.name} #{id} (#{state})"
|
|
227
|
+
return [] if completed? || aborted?
|
|
228
|
+
|
|
229
|
+
proposals = (input_categories.to_a + output_categories.to_a).filter_map do |category|
|
|
230
|
+
label = "#{where}, #{self.class.rocketjob_category_label(category)}"
|
|
231
|
+
next category.schema_proposal(label) unless category.is_a?(RocketJob::Category::Input)
|
|
232
|
+
|
|
233
|
+
# A job keeps the count of the records that it uploaded, which a scheduled job's next run does not copy.
|
|
234
|
+
defined = self.class.rocketjob_defined_category(category) || RocketJob::Category::Input.new(name: category.name)
|
|
235
|
+
category.schema_proposal(label, defined: defined, uploaded: record_count.to_i.positive?)
|
|
236
|
+
end
|
|
237
|
+
return proposals if queued? || proposals.empty?
|
|
238
|
+
|
|
239
|
+
[
|
|
240
|
+
RocketJob::SchemaProposal.new(
|
|
241
|
+
where: where,
|
|
242
|
+
notes: ["It holds deprecated category options, with which it has already uploaded its input. Let it " \
|
|
243
|
+
"finish, or abort it, before upgrading to Rocket Job 8, rather than change it."]
|
|
244
|
+
)
|
|
245
|
+
]
|
|
246
|
+
rescue StandardError => e
|
|
247
|
+
[RocketJob::SchemaProposal.unreadable(where, e)]
|
|
248
|
+
end
|
|
249
|
+
|
|
250
|
+
# Sets the supplied properties of each input category on it, such as those that a Dirmon entry or an
|
|
251
|
+
# UploadFileJob holds, and reports a deprecation warning for a deprecated option among them, see
|
|
252
|
+
# Category::Base.warn_deprecated_options.
|
|
123
253
|
def merge_input_categories(categories)
|
|
124
254
|
return if categories.blank?
|
|
125
255
|
|
|
126
256
|
categories.each do |properties|
|
|
127
257
|
category_name = (properties["name"] || properties[:name] || :main).to_sym
|
|
128
258
|
category = input_category(category_name)
|
|
129
|
-
|
|
259
|
+
rocketjob_categories_merge(category, properties)
|
|
130
260
|
end
|
|
131
261
|
end
|
|
132
262
|
|
|
263
|
+
# Sets the supplied properties of each output category on it, see #merge_input_categories.
|
|
133
264
|
def merge_output_categories(categories)
|
|
134
265
|
return if categories.blank?
|
|
135
266
|
|
|
136
267
|
categories.each do |properties|
|
|
137
268
|
category_name = (properties["name"] || properties[:name] || :main).to_sym
|
|
138
269
|
category = output_category(category_name)
|
|
139
|
-
|
|
270
|
+
rocketjob_categories_merge(category, properties)
|
|
140
271
|
end
|
|
141
272
|
end
|
|
142
273
|
|
|
143
274
|
private
|
|
144
275
|
|
|
276
|
+
# Sets the supplied properties on the category, after reporting a deprecation warning for a deprecated option
|
|
277
|
+
# among them, which includes the category's schema, since an input category's columns are not deprecated with one.
|
|
278
|
+
def rocketjob_categories_merge(category, properties)
|
|
279
|
+
category.class.warn_deprecated_options(category.attributes.slice("schema").merge(properties.to_h.stringify_keys))
|
|
280
|
+
properties.each { |key, value| category.public_send(:"#{key}=", value) }
|
|
281
|
+
end
|
|
282
|
+
|
|
145
283
|
def rocketjob_categories_assign
|
|
146
|
-
|
|
147
|
-
if input_categories.empty?
|
|
148
|
-
self.input_categories =
|
|
149
|
-
if self.class.defined_input_categories
|
|
150
|
-
self.class.defined_input_categories.deep_dup
|
|
151
|
-
else
|
|
152
|
-
[RocketJob::Category::Input.new]
|
|
153
|
-
end
|
|
154
|
-
end
|
|
284
|
+
self.input_categories = self.class.rocketjob_initial_input_categories.deep_dup if input_categories.empty?
|
|
155
285
|
|
|
156
286
|
return if !output_categories.empty? || !self.class.defined_output_categories
|
|
157
287
|
|
|
@@ -180,18 +310,9 @@ module RocketJob
|
|
|
180
310
|
return if row.nil?
|
|
181
311
|
|
|
182
312
|
category = input_category
|
|
183
|
-
return row if category.nil?
|
|
184
|
-
return nil if row.blank?
|
|
185
|
-
|
|
186
|
-
tabular = category.tabular
|
|
187
|
-
|
|
188
|
-
# Return the row as-is if the required header has not yet been set.
|
|
189
|
-
if tabular.header?
|
|
190
|
-
raise(ArgumentError,
|
|
191
|
-
"The tabular header columns _must_ be set before attempting to parse data that requires it.")
|
|
192
|
-
end
|
|
313
|
+
return row if category.nil?
|
|
193
314
|
|
|
194
|
-
|
|
315
|
+
category.parse(row)
|
|
195
316
|
end
|
|
196
317
|
|
|
197
318
|
def rocketjob_categories_output_render_row(row)
|
|
@@ -218,7 +339,7 @@ module RocketJob
|
|
|
218
339
|
|
|
219
340
|
# Migrate existing v5 batch jobs to v6
|
|
220
341
|
def rocketjob_categories_migrate
|
|
221
|
-
return unless
|
|
342
|
+
return unless attributes["input_categories"]&.first.is_a?(Symbol)
|
|
222
343
|
|
|
223
344
|
serializer = :none
|
|
224
345
|
if attribute_present?(:compress)
|