rocketjob 7.1.0 → 7.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/rocket_job/batch/categories.rb +145 -37
- data/lib/rocket_job/batch/io.rb +222 -141
- data/lib/rocket_job/category/base.rb +57 -0
- data/lib/rocket_job/category/input.rb +252 -62
- data/lib/rocket_job/category/input_schema_header.rb +90 -0
- data/lib/rocket_job/category/input_schema_proposal.rb +221 -0
- data/lib/rocket_job/category/input_text_lines.rb +37 -0
- data/lib/rocket_job/category/output.rb +32 -0
- data/lib/rocket_job/deprecation.rb +12 -2
- data/lib/rocket_job/dirmon_entry.rb +126 -6
- data/lib/rocket_job/extensions/iostreams/path.rb +24 -0
- data/lib/rocket_job/jobs/copy_file_job.rb +103 -19
- data/lib/rocket_job/jobs/dirmon_job.rb +7 -4
- data/lib/rocket_job/jobs/on_demand_batch_job.rb +118 -4
- data/lib/rocket_job/jobs/upload_file_job.rb +57 -3
- data/lib/rocket_job/plugins/job/model.rb +55 -0
- data/lib/rocket_job/rocket_job.rb +33 -0
- data/lib/rocket_job/schema_proposal.rb +232 -0
- data/lib/rocket_job/sliced/input.rb +149 -5
- data/lib/rocket_job/sliced/slice.rb +4 -0
- data/lib/rocket_job/sliced/slices.rb +1 -1
- data/lib/rocket_job/sliced/writer/input.rb +9 -2
- data/lib/rocket_job/version.rb +1 -1
- data/lib/rocketjob.rb +4 -0
- metadata +6 -2
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 60354ecdc42ea57a95256cfb2e1c239ced4449fbf5acd99d6707c7d2948fb06d
|
|
4
|
+
data.tar.gz: e99ef1bddca30c09cf7496c4cdca33b5360a56542230b076541e89ce50fb7aa6
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 529c1e43856e754b4204a3b2f6a24f1b0bcda0e90e19ee980637372d420c97b8ee97213d7d1564e5d5c5f1889ca831faed4d6e765f77b09df81bcaab92f83598
|
|
7
|
+
data.tar.gz: 2e1d359ba192068dad4532b491f796d20352c2263d32572f04b6f684e5b328aff1f1fb413566d3adadc67f92528bacd3143daee1ced7cf90c065743f61751c2f
|
|
@@ -26,28 +26,24 @@ module RocketJob
|
|
|
26
26
|
end
|
|
27
27
|
|
|
28
28
|
module ClassMethods
|
|
29
|
-
# Define a new input category
|
|
29
|
+
# Define a new input category, or replace the input category with the same name, and return it.
|
|
30
|
+
# A subclass that defines a category does not change the categories of its parent class.
|
|
30
31
|
# @see RocketJob::Category::Input
|
|
31
32
|
def input_category(**args)
|
|
32
33
|
RocketJob::Category::Input.warn_deprecated_options(args)
|
|
33
|
-
category
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
else
|
|
37
|
-
rocketjob_categories_set(category, defined_input_categories)
|
|
38
|
-
end
|
|
34
|
+
category = RocketJob::Category::Input.new(**args)
|
|
35
|
+
self.defined_input_categories = rocketjob_categories_with(defined_input_categories, category)
|
|
36
|
+
category
|
|
39
37
|
end
|
|
40
38
|
|
|
41
|
-
# Define a new output category
|
|
39
|
+
# Define a new output category, or replace the output category with the same name, and return it.
|
|
40
|
+
# A subclass that defines a category does not change the categories of its parent class.
|
|
42
41
|
# @see RocketJob::Category::Output
|
|
43
42
|
def output_category(**args)
|
|
44
43
|
RocketJob::Category::Output.warn_deprecated_options(args)
|
|
45
|
-
category
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
else
|
|
49
|
-
rocketjob_categories_set(category, defined_output_categories)
|
|
50
|
-
end
|
|
44
|
+
category = RocketJob::Category::Output.new(**args)
|
|
45
|
+
self.defined_output_categories = rocketjob_categories_with(defined_output_categories, category)
|
|
46
|
+
category
|
|
51
47
|
end
|
|
52
48
|
|
|
53
49
|
# Builds this job instance from the supplied properties hash that may contain input and output categories.
|
|
@@ -64,12 +60,111 @@ module RocketJob
|
|
|
64
60
|
job
|
|
65
61
|
end
|
|
66
62
|
|
|
63
|
+
# Returns [Array<RocketJob::SchemaProposal>] the schemas proposed to replace the deprecated options of the
|
|
64
|
+
# categories that this job class defines, rather than inherits, see RocketJob::Category::Input#schema_proposal.
|
|
65
|
+
# A class holds the same category as its parent class for each one that it inherits, see
|
|
66
|
+
# #rocketjob_categories_with, and a new one for each that it defines or redefines.
|
|
67
|
+
def schema_proposals
|
|
68
|
+
inherited = superclass.respond_to?(:defined_input_categories) ? rocketjob_categories_of(superclass) : []
|
|
69
|
+
rocketjob_categories_of(self).filter_map do |category|
|
|
70
|
+
next if inherited.any? { |parent| parent.equal?(category) }
|
|
71
|
+
|
|
72
|
+
category.schema_proposal("#{name}, #{rocketjob_category_label(category)}")
|
|
73
|
+
end
|
|
74
|
+
end
|
|
75
|
+
|
|
76
|
+
# Returns [true] since a saved job of this class can hold deprecated category options, see
|
|
77
|
+
# RocketJob::Plugins::Job::Model::ClassMethods#proposes_schemas?.
|
|
78
|
+
def proposes_schemas?
|
|
79
|
+
true
|
|
80
|
+
end
|
|
81
|
+
|
|
82
|
+
# Returns [Array<RocketJob::SchemaProposal>] the schemas proposed to replace the deprecated options in the
|
|
83
|
+
# supplied properties of a job of this class, such as those that a Dirmon entry or an UploadFileJob holds, at
|
|
84
|
+
# the supplied place. Each is proposed for the category that the properties build, with the job class's own
|
|
85
|
+
# settings, or says that a job of this class has no category of that name, for which #from_properties raises.
|
|
86
|
+
def property_schema_proposals(properties, where)
|
|
87
|
+
properties = properties.to_h.stringify_keys
|
|
88
|
+
{"input_categories" => RocketJob::Category::Input, "output_categories" => RocketJob::Category::Output}.flat_map do |key, klass|
|
|
89
|
+
Array(properties[key]).filter_map do |category_properties|
|
|
90
|
+
next unless category_properties.is_a?(Hash)
|
|
91
|
+
|
|
92
|
+
category_properties = category_properties.stringify_keys
|
|
93
|
+
next if klass.deprecated_options(category_properties).empty?
|
|
94
|
+
|
|
95
|
+
category = klass.new(name: (category_properties["name"] || :main).to_sym)
|
|
96
|
+
label = "#{where}, #{rocketjob_category_label(category)}, in its properties"
|
|
97
|
+
built = rocketjob_category_from(category, category_properties)
|
|
98
|
+
next built.schema_proposal(label) if built
|
|
99
|
+
|
|
100
|
+
RocketJob::SchemaProposal.new(
|
|
101
|
+
where: label,
|
|
102
|
+
error: "#{name} has no #{rocketjob_category_label(category)}, so a job cannot be built from these " \
|
|
103
|
+
"properties. Remove them, or define that category in the job class."
|
|
104
|
+
)
|
|
105
|
+
end
|
|
106
|
+
end
|
|
107
|
+
end
|
|
108
|
+
|
|
109
|
+
# Returns [String] how a category is named in a proposal, such as "input category main".
|
|
110
|
+
def rocketjob_category_label(category)
|
|
111
|
+
"#{category.class.name.demodulize.downcase} category #{category.name}"
|
|
112
|
+
end
|
|
113
|
+
|
|
114
|
+
# Returns [RocketJob::Category::Input|RocketJob::Category::Output] the category of the supplied class that a job
|
|
115
|
+
# of this class starts with, by the name of the supplied category, see #rocketjob_categories_assign, or nil when
|
|
116
|
+
# it starts without one.
|
|
117
|
+
def rocketjob_defined_category(category)
|
|
118
|
+
defined = category.is_a?(RocketJob::Category::Input) ? rocketjob_initial_input_categories : defined_output_categories
|
|
119
|
+
defined&.find { |candidate| candidate.name == category.name }
|
|
120
|
+
end
|
|
121
|
+
|
|
122
|
+
# Returns [Array<RocketJob::Category::Input>] the input categories that a job of this class starts with: those
|
|
123
|
+
# that it defines, or the main one when it defines none. Copy them before changing them.
|
|
124
|
+
def rocketjob_initial_input_categories
|
|
125
|
+
defined_input_categories || [RocketJob::Category::Input.new]
|
|
126
|
+
end
|
|
127
|
+
|
|
128
|
+
# Returns [Hash] the supplied attributes or properties of this job class to show to people, see
|
|
129
|
+
# RocketJob::Plugins::Job::Model.display_properties, with each of its input and output categories also
|
|
130
|
+
# showing its file name without any credentials.
|
|
131
|
+
def display_properties(properties)
|
|
132
|
+
super.to_h do |key, value|
|
|
133
|
+
category_class =
|
|
134
|
+
case key.to_s
|
|
135
|
+
when "input_categories" then RocketJob::Category::Input
|
|
136
|
+
when "output_categories" then RocketJob::Category::Output
|
|
137
|
+
end
|
|
138
|
+
next [key, value] unless category_class && value.is_a?(Array)
|
|
139
|
+
|
|
140
|
+
[key, value.map { |category| category.is_a?(Hash) ? category_class.display_properties(category) : category }]
|
|
141
|
+
end
|
|
142
|
+
end
|
|
143
|
+
|
|
67
144
|
private
|
|
68
145
|
|
|
69
|
-
def
|
|
70
|
-
|
|
146
|
+
def rocketjob_categories_of(klass)
|
|
147
|
+
[*klass.defined_input_categories, *klass.defined_output_categories]
|
|
148
|
+
end
|
|
149
|
+
|
|
150
|
+
# Returns the category that the supplied properties build for a job of this class, from its own category of
|
|
151
|
+
# the class and name of the supplied one, see #merge_input_categories, without reporting a deprecation warning,
|
|
152
|
+
# or nil when a job of this class has no such category.
|
|
153
|
+
def rocketjob_category_from(category, properties)
|
|
154
|
+
built = rocketjob_defined_category(category)&.deep_dup
|
|
155
|
+
properties.each_pair { |key, value| built.public_send(:"#{key}=", value) } if built
|
|
156
|
+
built
|
|
157
|
+
end
|
|
158
|
+
|
|
159
|
+
# Returns [Array] a copy of the supplied categories, which can be nil, with the supplied category in place of
|
|
160
|
+
# the one with the same name, or after them. The supplied categories are not changed, since a class shares
|
|
161
|
+
# them with its subclasses. Only the list is copied, not the categories in it, so that .schema_proposals can
|
|
162
|
+
# tell the categories that a class inherits from those that it defines.
|
|
163
|
+
def rocketjob_categories_with(categories, category)
|
|
164
|
+
categories = categories ? categories.dup : []
|
|
165
|
+
index = categories.find_index { |cat| cat.name == category.name }
|
|
71
166
|
index ? categories[index] = category : categories << category
|
|
72
|
-
|
|
167
|
+
categories
|
|
73
168
|
end
|
|
74
169
|
end
|
|
75
170
|
|
|
@@ -122,6 +217,36 @@ module RocketJob
|
|
|
122
217
|
false
|
|
123
218
|
end
|
|
124
219
|
|
|
220
|
+
# Returns [Array<RocketJob::SchemaProposal>] the schemas proposed to replace the deprecated options of this saved
|
|
221
|
+
# job's categories, see RocketJob::Category::Input#schema_proposal. A job that is running, paused or failed has
|
|
222
|
+
# uploaded its input with them, so it is only told to finish first. None for a job that has completed or was
|
|
223
|
+
# aborted, since it cannot run again. What reading its categories raises is answered as one proposal with the
|
|
224
|
+
# error, see RocketJob::SchemaProposal.unreadable.
|
|
225
|
+
def schema_proposals
|
|
226
|
+
where = "#{self.class.name} #{id} (#{state})"
|
|
227
|
+
return [] if completed? || aborted?
|
|
228
|
+
|
|
229
|
+
proposals = (input_categories.to_a + output_categories.to_a).filter_map do |category|
|
|
230
|
+
label = "#{where}, #{self.class.rocketjob_category_label(category)}"
|
|
231
|
+
next category.schema_proposal(label) unless category.is_a?(RocketJob::Category::Input)
|
|
232
|
+
|
|
233
|
+
# A job keeps the count of the records that it uploaded, which a scheduled job's next run does not copy.
|
|
234
|
+
defined = self.class.rocketjob_defined_category(category) || RocketJob::Category::Input.new(name: category.name)
|
|
235
|
+
category.schema_proposal(label, defined: defined, uploaded: record_count.to_i.positive?)
|
|
236
|
+
end
|
|
237
|
+
return proposals if queued? || proposals.empty?
|
|
238
|
+
|
|
239
|
+
[
|
|
240
|
+
RocketJob::SchemaProposal.new(
|
|
241
|
+
where: where,
|
|
242
|
+
notes: ["It holds deprecated category options, with which it has already uploaded its input. Let it " \
|
|
243
|
+
"finish, or abort it, before upgrading to Rocket Job 8, rather than change it."]
|
|
244
|
+
)
|
|
245
|
+
]
|
|
246
|
+
rescue StandardError => e
|
|
247
|
+
[RocketJob::SchemaProposal.unreadable(where, e)]
|
|
248
|
+
end
|
|
249
|
+
|
|
125
250
|
# Sets the supplied properties of each input category on it, such as those that a Dirmon entry or an
|
|
126
251
|
# UploadFileJob holds, and reports a deprecation warning for a deprecated option among them, see
|
|
127
252
|
# Category::Base.warn_deprecated_options.
|
|
@@ -156,15 +281,7 @@ module RocketJob
|
|
|
156
281
|
end
|
|
157
282
|
|
|
158
283
|
def rocketjob_categories_assign
|
|
159
|
-
|
|
160
|
-
if input_categories.empty?
|
|
161
|
-
self.input_categories =
|
|
162
|
-
if self.class.defined_input_categories
|
|
163
|
-
self.class.defined_input_categories.deep_dup
|
|
164
|
-
else
|
|
165
|
-
[RocketJob::Category::Input.new]
|
|
166
|
-
end
|
|
167
|
-
end
|
|
284
|
+
self.input_categories = self.class.rocketjob_initial_input_categories.deep_dup if input_categories.empty?
|
|
168
285
|
|
|
169
286
|
return if !output_categories.empty? || !self.class.defined_output_categories
|
|
170
287
|
|
|
@@ -193,18 +310,9 @@ module RocketJob
|
|
|
193
310
|
return if row.nil?
|
|
194
311
|
|
|
195
312
|
category = input_category
|
|
196
|
-
return row if category.nil?
|
|
197
|
-
return nil if row.blank?
|
|
198
|
-
|
|
199
|
-
tabular = category.tabular
|
|
200
|
-
|
|
201
|
-
# Return the row as-is if the required header has not yet been set.
|
|
202
|
-
if tabular.header?
|
|
203
|
-
raise(ArgumentError,
|
|
204
|
-
"The tabular header columns _must_ be set before attempting to parse data that requires it.")
|
|
205
|
-
end
|
|
313
|
+
return row if category.nil?
|
|
206
314
|
|
|
207
|
-
|
|
315
|
+
category.parse(row)
|
|
208
316
|
end
|
|
209
317
|
|
|
210
318
|
def rocketjob_categories_output_render_row(row)
|