rocketjob 7.0.0 → 7.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 7cf0e18a83d97aadac0d2741a399bcb55fa33884ccf2660836a5a3cfd946f772
4
- data.tar.gz: d95563d0c77495dcdb3498dccf7cfc22fefee78bd22df244b823fc3939ef5373
3
+ metadata.gz: 60354ecdc42ea57a95256cfb2e1c239ced4449fbf5acd99d6707c7d2948fb06d
4
+ data.tar.gz: e99ef1bddca30c09cf7496c4cdca33b5360a56542230b076541e89ce50fb7aa6
5
5
  SHA512:
6
- metadata.gz: 8f379b7890ddce715719616c311b501a916bc081bfd8eaba5055fa124badd58084985d9f38da0c081a8b5bcff3c67108ba3ea5ca52d8f2a8d7a92aa7297c94e8
7
- data.tar.gz: db4e047d9c07faed020db569f7268fc0dea981ede5792ec40b7636718b118514fc6edc8d08b961d3dfe1d4ecadada40f548dde490ed4ede5326bbcb12ef5bedd
6
+ metadata.gz: 529c1e43856e754b4204a3b2f6a24f1b0bcda0e90e19ee980637372d420c97b8ee97213d7d1564e5d5c5f1889ca831faed4d6e765f77b09df81bcaab92f83598
7
+ data.tar.gz: 2e1d359ba192068dad4532b491f796d20352c2263d32572f04b6f684e5b328aff1f1fb413566d3adadc67f92528bacd3143daee1ced7cf90c065743f61751c2f
data/README.md CHANGED
@@ -1,5 +1,5 @@
1
1
  # Rocket Job
2
- [![Gem Version](https://img.shields.io/gem/v/rocketjob.svg)](https://rubygems.org/gems/rocketjob) [![Downloads](https://img.shields.io/gem/dt/rocketjob.svg)](https://rubygems.org/gems/rocketjob) [![License](https://img.shields.io/badge/license-Apache%202.0-brightgreen.svg)](http://opensource.org/licenses/Apache-2.0) ![](https://img.shields.io/badge/status-Production%20Ready-blue.svg)
2
+ [![Gem Version](https://img.shields.io/gem/v/rocketjob.svg)](https://rubygems.org/gems/rocketjob) [![Downloads](https://img.shields.io/gem/dt/rocketjob.svg)](https://rubygems.org/gems/rocketjob) [![Coverage](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/reidmorrison/rocketjob/badges/coverage.json)](https://github.com/reidmorrison/rocketjob/actions/workflows/ci.yml) [![License](https://img.shields.io/badge/license-Apache%202.0-brightgreen.svg)](http://opensource.org/licenses/Apache-2.0) ![](https://img.shields.io/badge/status-Production%20Ready-blue.svg)
3
3
 
4
4
  **Process millions of records across thousands of workers.**
5
5
 
@@ -26,26 +26,24 @@ module RocketJob
26
26
  end
27
27
 
28
28
  module ClassMethods
29
- # Define a new input category
29
+ # Define a new input category, or replace the input category with the same name, and return it.
30
+ # A subclass that defines a category does not change the categories of its parent class.
30
31
  # @see RocketJob::Category::Input
31
32
  def input_category(**args)
32
- category = RocketJob::Category::Input.new(**args)
33
- if defined_input_categories.nil?
34
- self.defined_input_categories = [category]
35
- else
36
- rocketjob_categories_set(category, defined_input_categories)
37
- end
33
+ RocketJob::Category::Input.warn_deprecated_options(args)
34
+ category = RocketJob::Category::Input.new(**args)
35
+ self.defined_input_categories = rocketjob_categories_with(defined_input_categories, category)
36
+ category
38
37
  end
39
38
 
40
- # Define a new output category
39
+ # Define a new output category, or replace the output category with the same name, and return it.
40
+ # A subclass that defines a category does not change the categories of its parent class.
41
41
  # @see RocketJob::Category::Output
42
42
  def output_category(**args)
43
- category = RocketJob::Category::Output.new(**args)
44
- if defined_output_categories.nil?
45
- self.defined_output_categories = [category]
46
- else
47
- rocketjob_categories_set(category, defined_output_categories)
48
- end
43
+ RocketJob::Category::Output.warn_deprecated_options(args)
44
+ category = RocketJob::Category::Output.new(**args)
45
+ self.defined_output_categories = rocketjob_categories_with(defined_output_categories, category)
46
+ category
49
47
  end
50
48
 
51
49
  # Builds this job instance from the supplied properties hash that may contain input and output categories.
@@ -62,12 +60,111 @@ module RocketJob
62
60
  job
63
61
  end
64
62
 
63
+ # Returns [Array<RocketJob::SchemaProposal>] the schemas proposed to replace the deprecated options of the
64
+ # categories that this job class defines, rather than inherits, see RocketJob::Category::Input#schema_proposal.
65
+ # A class holds the same category as its parent class for each one that it inherits, see
66
+ # #rocketjob_categories_with, and a new one for each that it defines or redefines.
67
+ def schema_proposals
68
+ inherited = superclass.respond_to?(:defined_input_categories) ? rocketjob_categories_of(superclass) : []
69
+ rocketjob_categories_of(self).filter_map do |category|
70
+ next if inherited.any? { |parent| parent.equal?(category) }
71
+
72
+ category.schema_proposal("#{name}, #{rocketjob_category_label(category)}")
73
+ end
74
+ end
75
+
76
+ # Returns [true] since a saved job of this class can hold deprecated category options, see
77
+ # RocketJob::Plugins::Job::Model::ClassMethods#proposes_schemas?.
78
+ def proposes_schemas?
79
+ true
80
+ end
81
+
82
+ # Returns [Array<RocketJob::SchemaProposal>] the schemas proposed to replace the deprecated options in the
83
+ # supplied properties of a job of this class, such as those that a Dirmon entry or an UploadFileJob holds, at
84
+ # the supplied place. Each is proposed for the category that the properties build, with the job class's own
85
+ # settings, or says that a job of this class has no category of that name, for which #from_properties raises.
86
+ def property_schema_proposals(properties, where)
87
+ properties = properties.to_h.stringify_keys
88
+ {"input_categories" => RocketJob::Category::Input, "output_categories" => RocketJob::Category::Output}.flat_map do |key, klass|
89
+ Array(properties[key]).filter_map do |category_properties|
90
+ next unless category_properties.is_a?(Hash)
91
+
92
+ category_properties = category_properties.stringify_keys
93
+ next if klass.deprecated_options(category_properties).empty?
94
+
95
+ category = klass.new(name: (category_properties["name"] || :main).to_sym)
96
+ label = "#{where}, #{rocketjob_category_label(category)}, in its properties"
97
+ built = rocketjob_category_from(category, category_properties)
98
+ next built.schema_proposal(label) if built
99
+
100
+ RocketJob::SchemaProposal.new(
101
+ where: label,
102
+ error: "#{name} has no #{rocketjob_category_label(category)}, so a job cannot be built from these " \
103
+ "properties. Remove them, or define that category in the job class."
104
+ )
105
+ end
106
+ end
107
+ end
108
+
109
+ # Returns [String] how a category is named in a proposal, such as "input category main".
110
+ def rocketjob_category_label(category)
111
+ "#{category.class.name.demodulize.downcase} category #{category.name}"
112
+ end
113
+
114
+ # Returns [RocketJob::Category::Input|RocketJob::Category::Output] the category of the supplied class that a job
115
+ # of this class starts with, by the name of the supplied category, see #rocketjob_categories_assign, or nil when
116
+ # it starts without one.
117
+ def rocketjob_defined_category(category)
118
+ defined = category.is_a?(RocketJob::Category::Input) ? rocketjob_initial_input_categories : defined_output_categories
119
+ defined&.find { |candidate| candidate.name == category.name }
120
+ end
121
+
122
+ # Returns [Array<RocketJob::Category::Input>] the input categories that a job of this class starts with: those
123
+ # that it defines, or the main one when it defines none. Copy them before changing them.
124
+ def rocketjob_initial_input_categories
125
+ defined_input_categories || [RocketJob::Category::Input.new]
126
+ end
127
+
128
+ # Returns [Hash] the supplied attributes or properties of this job class to show to people, see
129
+ # RocketJob::Plugins::Job::Model.display_properties, with each of its input and output categories also
130
+ # showing its file name without any credentials.
131
+ def display_properties(properties)
132
+ super.to_h do |key, value|
133
+ category_class =
134
+ case key.to_s
135
+ when "input_categories" then RocketJob::Category::Input
136
+ when "output_categories" then RocketJob::Category::Output
137
+ end
138
+ next [key, value] unless category_class && value.is_a?(Array)
139
+
140
+ [key, value.map { |category| category.is_a?(Hash) ? category_class.display_properties(category) : category }]
141
+ end
142
+ end
143
+
65
144
  private
66
145
 
67
- def rocketjob_categories_set(category, categories)
68
- index = categories.find_index { |cat| cat.name == category.name }
146
+ def rocketjob_categories_of(klass)
147
+ [*klass.defined_input_categories, *klass.defined_output_categories]
148
+ end
149
+
150
+ # Returns the category that the supplied properties build for a job of this class, from its own category of
151
+ # the class and name of the supplied one, see #merge_input_categories, without reporting a deprecation warning,
152
+ # or nil when a job of this class has no such category.
153
+ def rocketjob_category_from(category, properties)
154
+ built = rocketjob_defined_category(category)&.deep_dup
155
+ properties.each_pair { |key, value| built.public_send(:"#{key}=", value) } if built
156
+ built
157
+ end
158
+
159
+ # Returns [Array] a copy of the supplied categories, which can be nil, with the supplied category in place of
160
+ # the one with the same name, or after them. The supplied categories are not changed, since a class shares
161
+ # them with its subclasses. Only the list is copied, not the categories in it, so that .schema_proposals can
162
+ # tell the categories that a class inherits from those that it defines.
163
+ def rocketjob_categories_with(categories, category)
164
+ categories = categories ? categories.dup : []
165
+ index = categories.find_index { |cat| cat.name == category.name }
69
166
  index ? categories[index] = category : categories << category
70
- category
167
+ categories
71
168
  end
72
169
  end
73
170
 
@@ -120,38 +217,71 @@ module RocketJob
120
217
  false
121
218
  end
122
219
 
220
+ # Returns [Array<RocketJob::SchemaProposal>] the schemas proposed to replace the deprecated options of this saved
221
+ # job's categories, see RocketJob::Category::Input#schema_proposal. A job that is running, paused or failed has
222
+ # uploaded its input with them, so it is only told to finish first. None for a job that has completed or was
223
+ # aborted, since it cannot run again. What reading its categories raises is answered as one proposal with the
224
+ # error, see RocketJob::SchemaProposal.unreadable.
225
+ def schema_proposals
226
+ where = "#{self.class.name} #{id} (#{state})"
227
+ return [] if completed? || aborted?
228
+
229
+ proposals = (input_categories.to_a + output_categories.to_a).filter_map do |category|
230
+ label = "#{where}, #{self.class.rocketjob_category_label(category)}"
231
+ next category.schema_proposal(label) unless category.is_a?(RocketJob::Category::Input)
232
+
233
+ # A job keeps the count of the records that it uploaded, which a scheduled job's next run does not copy.
234
+ defined = self.class.rocketjob_defined_category(category) || RocketJob::Category::Input.new(name: category.name)
235
+ category.schema_proposal(label, defined: defined, uploaded: record_count.to_i.positive?)
236
+ end
237
+ return proposals if queued? || proposals.empty?
238
+
239
+ [
240
+ RocketJob::SchemaProposal.new(
241
+ where: where,
242
+ notes: ["It holds deprecated category options, with which it has already uploaded its input. Let it " \
243
+ "finish, or abort it, before upgrading to Rocket Job 8, rather than change it."]
244
+ )
245
+ ]
246
+ rescue StandardError => e
247
+ [RocketJob::SchemaProposal.unreadable(where, e)]
248
+ end
249
+
250
+ # Sets the supplied properties of each input category on it, such as those that a Dirmon entry or an
251
+ # UploadFileJob holds, and reports a deprecation warning for a deprecated option among them, see
252
+ # Category::Base.warn_deprecated_options.
123
253
  def merge_input_categories(categories)
124
254
  return if categories.blank?
125
255
 
126
256
  categories.each do |properties|
127
257
  category_name = (properties["name"] || properties[:name] || :main).to_sym
128
258
  category = input_category(category_name)
129
- properties.each { |key, value| category.public_send(:"#{key}=", value) }
259
+ rocketjob_categories_merge(category, properties)
130
260
  end
131
261
  end
132
262
 
263
+ # Sets the supplied properties of each output category on it, see #merge_input_categories.
133
264
  def merge_output_categories(categories)
134
265
  return if categories.blank?
135
266
 
136
267
  categories.each do |properties|
137
268
  category_name = (properties["name"] || properties[:name] || :main).to_sym
138
269
  category = output_category(category_name)
139
- properties.each { |key, value| category.public_send(:"#{key}=", value) }
270
+ rocketjob_categories_merge(category, properties)
140
271
  end
141
272
  end
142
273
 
143
274
  private
144
275
 
276
+ # Sets the supplied properties on the category, after reporting a deprecation warning for a deprecated option
277
+ # among them, which includes the category's schema, since an input category's columns are not deprecated with one.
278
+ def rocketjob_categories_merge(category, properties)
279
+ category.class.warn_deprecated_options(category.attributes.slice("schema").merge(properties.to_h.stringify_keys))
280
+ properties.each { |key, value| category.public_send(:"#{key}=", value) }
281
+ end
282
+
145
283
  def rocketjob_categories_assign
146
- # Input categories defaults to :main if none was set in the class
147
- if input_categories.empty?
148
- self.input_categories =
149
- if self.class.defined_input_categories
150
- self.class.defined_input_categories.deep_dup
151
- else
152
- [RocketJob::Category::Input.new]
153
- end
154
- end
284
+ self.input_categories = self.class.rocketjob_initial_input_categories.deep_dup if input_categories.empty?
155
285
 
156
286
  return if !output_categories.empty? || !self.class.defined_output_categories
157
287
 
@@ -180,18 +310,9 @@ module RocketJob
180
310
  return if row.nil?
181
311
 
182
312
  category = input_category
183
- return row if category.nil? || !category.tabular?
184
- return nil if row.blank?
185
-
186
- tabular = category.tabular
187
-
188
- # Return the row as-is if the required header has not yet been set.
189
- if tabular.header?
190
- raise(ArgumentError,
191
- "The tabular header columns _must_ be set before attempting to parse data that requires it.")
192
- end
313
+ return row if category.nil?
193
314
 
194
- tabular.record_parse(row)
315
+ category.parse(row)
195
316
  end
196
317
 
197
318
  def rocketjob_categories_output_render_row(row)
@@ -218,7 +339,7 @@ module RocketJob
218
339
 
219
340
  # Migrate existing v5 batch jobs to v6
220
341
  def rocketjob_categories_migrate
221
- return unless attribute_present?(:input_categories) && self[:input_categories]&.first.is_a?(Symbol)
342
+ return unless attributes["input_categories"]&.first.is_a?(Symbol)
222
343
 
223
344
  serializer = :none
224
345
  if attribute_present?(:compress)