rocketjob 7.1.0 → 7.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/rocket_job/batch/categories.rb +145 -37
- data/lib/rocket_job/batch/io.rb +222 -141
- data/lib/rocket_job/category/base.rb +57 -0
- data/lib/rocket_job/category/input.rb +252 -62
- data/lib/rocket_job/category/input_schema_header.rb +90 -0
- data/lib/rocket_job/category/input_schema_proposal.rb +221 -0
- data/lib/rocket_job/category/input_text_lines.rb +37 -0
- data/lib/rocket_job/category/output.rb +32 -0
- data/lib/rocket_job/deprecation.rb +12 -2
- data/lib/rocket_job/dirmon_entry.rb +126 -6
- data/lib/rocket_job/extensions/iostreams/path.rb +24 -0
- data/lib/rocket_job/jobs/copy_file_job.rb +103 -19
- data/lib/rocket_job/jobs/dirmon_job.rb +7 -4
- data/lib/rocket_job/jobs/on_demand_batch_job.rb +118 -4
- data/lib/rocket_job/jobs/upload_file_job.rb +57 -3
- data/lib/rocket_job/plugins/job/model.rb +55 -0
- data/lib/rocket_job/rocket_job.rb +33 -0
- data/lib/rocket_job/schema_proposal.rb +232 -0
- data/lib/rocket_job/sliced/input.rb +149 -5
- data/lib/rocket_job/sliced/slice.rb +4 -0
- data/lib/rocket_job/sliced/slices.rb +1 -1
- data/lib/rocket_job/sliced/writer/input.rb +9 -2
- data/lib/rocket_job/version.rb +1 -1
- data/lib/rocketjob.rb +4 -0
- metadata +6 -2
|
@@ -5,6 +5,9 @@ module RocketJob
|
|
|
5
5
|
include SemanticLogger::Loggable
|
|
6
6
|
include Plugins::Document
|
|
7
7
|
include Category::Base
|
|
8
|
+
include Category::InputSchemaProposal
|
|
9
|
+
include Category::InputSchemaHeader
|
|
10
|
+
include Category::InputTextLines
|
|
8
11
|
|
|
9
12
|
embedded_in :job, class_name: "RocketJob::Job", inverse_of: :input_categories
|
|
10
13
|
|
|
@@ -49,22 +52,51 @@ module RocketJob
|
|
|
49
52
|
field :skip_unknown, type: ::Mongoid::Boolean, default: false
|
|
50
53
|
validates_inclusion_of :skip_unknown, in: [true, false]
|
|
51
54
|
|
|
52
|
-
#
|
|
53
|
-
#
|
|
55
|
+
# Whether `#upload` parses each line of a file in this category's format, see #format, as it uploads it, so that a
|
|
56
|
+
# file with a line that cannot be read, or with a value that is not valid for its field, see #schema, uploads
|
|
57
|
+
# nothing. The lines are still uploaded as they are, and parsed again by the workers, see #parse.
|
|
58
|
+
#
|
|
59
|
+
# Use it when the entire file must be validated before processing is started, and upload time is not important,
|
|
60
|
+
# since the lines are parsed one after another while the file is uploaded.
|
|
61
|
+
#
|
|
62
|
+
# Needs a format. Only a file is validated: not a query or a range, which upload values rather than the lines of
|
|
63
|
+
# a file, nor the records that the block of an upload writes, see RocketJob::Batch::IO#upload.
|
|
64
|
+
#
|
|
65
|
+
# Replaces the :hash `mode`, which is deprecated, see #mode, and is the same option in Rocket Job 8.
|
|
66
|
+
#
|
|
67
|
+
# Default: nil, which validates the lines of a category with a schema in the :hash `mode`, as 7.1 did. Set it to
|
|
68
|
+
# false so that such a category does not validate them.
|
|
69
|
+
field :validate_rows, type: ::Mongoid::Boolean
|
|
70
|
+
validate :validate_rows_has_format
|
|
71
|
+
|
|
72
|
+
# Why `validate_rows` cannot be true without a format, see #validate_rows_without_format?.
|
|
73
|
+
VALIDATE_ROWS_NEEDS_FORMAT = "needs a format, whose lines it validates".freeze
|
|
74
|
+
private_constant :VALIDATE_ROWS_NEEDS_FORMAT
|
|
75
|
+
|
|
76
|
+
# The options of an upload, see RocketJob::Batch::IO#upload, that this category reads a file with, see
|
|
77
|
+
# #each_record, which describe the columns of the file, and are deprecated: the schema replaces them, see #schema.
|
|
78
|
+
DEPRECATED_UPLOAD_OPTIONS = %i[allowed_columns required_columns skip_unknown cleanse_header].freeze
|
|
79
|
+
|
|
80
|
+
# The modes of an input category that are deprecated, see #mode and .warn_deprecated_mode.
|
|
81
|
+
DEPRECATED_MODES = %w[array hash].freeze
|
|
82
|
+
|
|
83
|
+
# How `#upload` reads a file into this category, unless the upload is given a `stream_mode:`, which is deprecated:
|
|
54
84
|
# :line
|
|
55
|
-
# Uploads the lines of the file, which the workers parse by the schema
|
|
56
|
-
# its type, such as a Date. This is the default, and the most performant, since
|
|
85
|
+
# Uploads the lines of the file, which the workers parse, by the schema when there is one, see #schema, so that
|
|
86
|
+
# each value reaches `#perform` as its type, such as a Date. This is the default, and the most performant, since
|
|
87
|
+
# the workers parse the lines.
|
|
57
88
|
# :hash
|
|
58
|
-
#
|
|
59
|
-
#
|
|
60
|
-
#
|
|
89
|
+
# Deprecated, use `validate_rows: true`, see #validate_rows, which Rocket Job 8 reads a saved :hash mode as.
|
|
90
|
+
# With a schema, see #schema, parses every line when it is uploaded, as `validate_rows: true` does. Without a
|
|
91
|
+
# schema it has no effect on how a file is read.
|
|
61
92
|
# :array
|
|
62
|
-
#
|
|
93
|
+
# Deprecated, use :line, and `validate_rows: true` to also parse every line when the file is uploaded. Cannot be
|
|
94
|
+
# used with a schema.
|
|
63
95
|
#
|
|
64
|
-
#
|
|
65
|
-
#
|
|
66
|
-
#
|
|
67
|
-
#
|
|
96
|
+
# Only the deprecated `stream_mode:` of an upload reads a file into records rather than lines: :hash without a
|
|
97
|
+
# format, and :array without a schema, see #each_record. Otherwise the lines of the file are uploaded, and its
|
|
98
|
+
# header row read, whatever this mode is, and the workers parse them, by the schema when there is one, so that
|
|
99
|
+
# each value reaches `#perform` as its type, such as a Date. The :array mode cannot be used with a schema.
|
|
68
100
|
field :mode, type: ::Mongoid::StringifiedSymbol, default: :line
|
|
69
101
|
validates_inclusion_of :mode, in: %i[line array hash]
|
|
70
102
|
|
|
@@ -89,6 +121,48 @@ module RocketJob
|
|
|
89
121
|
|
|
90
122
|
validates_inclusion_of :serializer, in: %i[none compress encrypt]
|
|
91
123
|
|
|
124
|
+
# Reports a deprecation warning, see RocketJob::Deprecation, for the deprecated options in the supplied
|
|
125
|
+
# properties of an input category, see Category::Base.warn_deprecated_options, and for its :array or :hash `mode`,
|
|
126
|
+
# see .warn_deprecated_mode.
|
|
127
|
+
def self.warn_deprecated_options(properties)
|
|
128
|
+
super
|
|
129
|
+
warn_deprecated_mode(properties.to_h.stringify_keys["mode"])
|
|
130
|
+
end
|
|
131
|
+
|
|
132
|
+
# Reports a deprecation warning, see RocketJob::Deprecation, for the supplied mode of an input category, see
|
|
133
|
+
# #mode, when it is deprecated: :array, and :hash, which `validate_rows` replaces, see #validate_rows.
|
|
134
|
+
def self.warn_deprecated_mode(mode)
|
|
135
|
+
case mode.to_s
|
|
136
|
+
when "array"
|
|
137
|
+
RocketJob::Deprecation.warn(
|
|
138
|
+
"mode: :array is deprecated, use :line, the default, which the workers parse, and validate_rows: true to " \
|
|
139
|
+
"also parse every line when the file is uploaded, see #{RocketJob::Deprecation::UPGRADE_GUIDE_V72}"
|
|
140
|
+
)
|
|
141
|
+
when "hash"
|
|
142
|
+
RocketJob::Deprecation.warn(
|
|
143
|
+
"mode: :hash is deprecated, use validate_rows: true instead, which parses every line of a file in the " \
|
|
144
|
+
"category's format when it is uploaded, as mode: :hash did with a schema. Without a schema mode: :hash " \
|
|
145
|
+
"has no effect, so remove it, since Rocket Job 8 reads it as validate_rows: true, see " \
|
|
146
|
+
"#{RocketJob::Deprecation::UPGRADE_GUIDE_V72}"
|
|
147
|
+
)
|
|
148
|
+
end
|
|
149
|
+
end
|
|
150
|
+
|
|
151
|
+
# Returns [Array<String>] the names of the deprecated options in the supplied properties of an input category,
|
|
152
|
+
# with String keys, see Category::Base.deprecated_options, and `mode` when it is :array or :hash, see #mode.
|
|
153
|
+
def self.deprecated_options(properties)
|
|
154
|
+
names = super
|
|
155
|
+
names << "mode" if DEPRECATED_MODES.include?(properties["mode"].to_s)
|
|
156
|
+
names
|
|
157
|
+
end
|
|
158
|
+
|
|
159
|
+
# Returns [Array<String>] the names of the deprecated options of an input category, see
|
|
160
|
+
# Category::Base.deprecated_option_names, and its column options, see .deprecated_column_options. Without `mode`,
|
|
161
|
+
# which is only deprecated for DEPRECATED_MODES.
|
|
162
|
+
def self.deprecated_option_names
|
|
163
|
+
super + %w[allowed_columns required_columns skip_unknown header_cleanser]
|
|
164
|
+
end
|
|
165
|
+
|
|
92
166
|
# Returns [Array<String>] the names of the deprecated options in the supplied properties of an input category,
|
|
93
167
|
# with String keys, see Category::Base.deprecated_column_options: `allowed_columns`, `required_columns`,
|
|
94
168
|
# `skip_unknown` when true, `header_cleanser` when :none, and `columns` without a schema, since with one they
|
|
@@ -157,38 +231,125 @@ module RocketJob
|
|
|
157
231
|
# unless an encoding is set on the supplied path, which reads it instead. A schema set on the supplied path
|
|
158
232
|
# is kept.
|
|
159
233
|
path.schema(schema) if path.schema.nil?
|
|
234
|
+
elsif format == :fixed
|
|
235
|
+
# A fixed width line keeps its length, so that its columns stay in place: each non-printable character, such
|
|
236
|
+
# as NUL padding or a tab, is replaced with a space, and so is each byte of a character that is not ASCII.
|
|
237
|
+
# The :printable cleaner would remove them, since it ignores `replace:`.
|
|
238
|
+
path.encoding("UTF-8", cleaner: :replace_non_printable, replace: " ")
|
|
160
239
|
elsif tabular?
|
|
161
|
-
# Remove non-printable characters from tabular input formats.
|
|
162
|
-
|
|
163
|
-
replace = format == :fixed ? " " : ""
|
|
164
|
-
path.encoding("UTF-8", cleaner: :printable, replace: replace)
|
|
240
|
+
# Remove non-printable characters, and characters that are not ASCII, from the other tabular input formats.
|
|
241
|
+
path.encoding("UTF-8", cleaner: :printable, replace: "")
|
|
165
242
|
end
|
|
166
243
|
path
|
|
167
244
|
end
|
|
168
245
|
|
|
169
|
-
#
|
|
170
|
-
#
|
|
246
|
+
# Returns the supplied record of an upload as `#perform` receives it: parsed in this category's format, or as it is
|
|
247
|
+
# when the category has no format. A blank record of a tabular format is nil.
|
|
171
248
|
#
|
|
172
|
-
#
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
mode ||= self.mode
|
|
179
|
-
if mode == :array
|
|
249
|
+
# Raises ArgumentError when the format needs a header row that has not been read yet.
|
|
250
|
+
def parse(record)
|
|
251
|
+
return record unless tabular?
|
|
252
|
+
return if record.blank?
|
|
253
|
+
|
|
254
|
+
if tabular.header?
|
|
180
255
|
raise(ArgumentError,
|
|
181
|
-
"
|
|
256
|
+
"The tabular header columns _must_ be set before attempting to parse data that requires it.")
|
|
257
|
+
end
|
|
258
|
+
|
|
259
|
+
tabular.record_parse(record)
|
|
260
|
+
end
|
|
261
|
+
|
|
262
|
+
# Returns [Array] the first records of the supplied file, or stream, as `#perform` would receive them if the file
|
|
263
|
+
# were uploaded into this category, see RocketJob::Batch::IO#upload. A copy of this category reads the file, so
|
|
264
|
+
# nothing is stored, and this category is not changed. Raises what the upload, or a worker parsing a record,
|
|
265
|
+
# would raise.
|
|
266
|
+
def preview(stream, limit: 100)
|
|
267
|
+
copy = deep_dup
|
|
268
|
+
# A copy of a path, so that the encoding that an upload sets on it does not change the caller's.
|
|
269
|
+
path = copy.upload_path(stream.is_a?(IOStreams::Stream) ? stream.dup : stream)
|
|
270
|
+
header = copy.extract_header_callback(nil)
|
|
271
|
+
records = []
|
|
272
|
+
# Each record is parsed below, as a worker parses it, so it is not also parsed while it is read.
|
|
273
|
+
copy.each_record(path, validate_rows: false) do |record|
|
|
274
|
+
if header
|
|
275
|
+
header = nil if header.call(record)
|
|
276
|
+
next
|
|
277
|
+
end
|
|
278
|
+
records << copy.parse(record)
|
|
279
|
+
break if records.size >= limit
|
|
182
280
|
end
|
|
183
|
-
|
|
281
|
+
records
|
|
282
|
+
end
|
|
283
|
+
|
|
284
|
+
# Yields each record read from the supplied path for an upload, see RocketJob::Batch::IO#upload: each line, which
|
|
285
|
+
# is first parsed in this category's format when it validates the rows of the file, see #upload_validates_rows?,
|
|
286
|
+
# with the supplied `validate_rows:` of the upload, so that a line that cannot be read, or a value that is not
|
|
287
|
+
# valid, raises before any record is processed.
|
|
288
|
+
#
|
|
289
|
+
# The supplied mode, the deprecated `stream_mode:` of the upload, decides alone whether the lines are parsed, unless
|
|
290
|
+
# it is nil or false: in the :hash mode each line is parsed first, as with `validate_rows: true`. It reads the file
|
|
291
|
+
# into records instead of lines, as 7.0 read it, see `IOStreams::Stream#each`, only where 7.0 could upload them,
|
|
292
|
+
# see #stream_mode_records?.
|
|
293
|
+
#
|
|
294
|
+
# In the :hash mode, the options of the upload that describe the columns of a file, which are deprecated, are
|
|
295
|
+
# left out where 7.1 accepted them, see #stream_mode_ignores_column_options?. Otherwise the line reader raises
|
|
296
|
+
# ArgumentError for them, as 7.1 did.
|
|
297
|
+
#
|
|
298
|
+
# Without a format, each line of a text file is uploaded as UTF-8, see #each_line.
|
|
299
|
+
def each_record(path, mode: nil, validate_rows: nil, **args, &)
|
|
300
|
+
mode = mode.presence&.to_sym
|
|
301
|
+
return path.each(mode, **args, &) if stream_mode_records?(mode)
|
|
302
|
+
|
|
303
|
+
if schema.present? && (mode || self.mode) == :array
|
|
304
|
+
raise(ArgumentError, "Input category #{name} has a schema, which cannot be uploaded in :array mode, use " \
|
|
305
|
+
":line, and validate_rows: true to also parse every line when the file is uploaded")
|
|
306
|
+
end
|
|
307
|
+
validate = mode ? mode == :hash : upload_validates_rows?(validate_rows)
|
|
308
|
+
return each_line(path, **args, &) unless validate
|
|
184
309
|
|
|
310
|
+
args.except!(*DEPRECATED_UPLOAD_OPTIONS) if stream_mode_ignores_column_options?(mode)
|
|
185
311
|
path.each(:line, **args) do |line|
|
|
186
|
-
# The lines before the header row, and the header row itself, are read by #extract_header_callback
|
|
187
|
-
|
|
312
|
+
# The lines before the header row, and the header row itself, are read by #extract_header_callback, and a
|
|
313
|
+
# blank line is not parsed, since a worker does not parse one either, see #parse.
|
|
314
|
+
tabular.record_parse(line) unless line.blank? || tabular.header?
|
|
188
315
|
yield(line)
|
|
189
316
|
end
|
|
190
317
|
end
|
|
191
318
|
|
|
319
|
+
# Returns [true|false] whether an upload of a file into this category parses each line of it as it uploads it,
|
|
320
|
+
# see #validate_rows: the supplied `validate_rows:` of the upload, unless it is nil, otherwise this category's,
|
|
321
|
+
# unless it is nil too, otherwise whether this category has a schema and the :hash `mode`, which 7.1 parsed each
|
|
322
|
+
# line in. Always false without a format, which has no lines to parse.
|
|
323
|
+
def upload_validates_rows?(validate_rows = nil)
|
|
324
|
+
return false unless tabular?
|
|
325
|
+
|
|
326
|
+
validate_rows = self.validate_rows if validate_rows.nil?
|
|
327
|
+
validate_rows.nil? ? schema.present? && mode == :hash : validate_rows
|
|
328
|
+
end
|
|
329
|
+
|
|
330
|
+
# Raises ArgumentError when a file cannot be uploaded into this category with the supplied `validate_rows:` of the
|
|
331
|
+
# upload, see #upload_validates_rows?: when it is not true, false or nil, or when it, or else this category's
|
|
332
|
+
# `validate_rows`, is true without a format, which has no lines to validate.
|
|
333
|
+
def check_upload(validate_rows = nil)
|
|
334
|
+
unless [nil, true, false].include?(validate_rows)
|
|
335
|
+
raise(ArgumentError, "Input category #{name}: validate_rows: must be true or false, not #{validate_rows.inspect}")
|
|
336
|
+
end
|
|
337
|
+
return unless validate_rows_without_format?(validate_rows.nil? ? self.validate_rows : validate_rows)
|
|
338
|
+
|
|
339
|
+
raise(ArgumentError, "Input category #{name}: validate_rows: #{VALIDATE_ROWS_NEEDS_FORMAT}, such as :auto, which " \
|
|
340
|
+
"reads a file in the format of its file name")
|
|
341
|
+
end
|
|
342
|
+
|
|
343
|
+
# Reports a deprecation warning, see RocketJob::Deprecation, for the supplied `stream_mode:` of an upload into
|
|
344
|
+
# this category, also nil, see RocketJob::Batch::IO#upload, which Rocket Job 8 removes, saying what replaces it
|
|
345
|
+
# here: a `stream_mode:` other than nil or false decides alone whether the lines are parsed, see #each_record.
|
|
346
|
+
def warn_deprecated_stream_mode(stream_mode)
|
|
347
|
+
RocketJob::Deprecation.warn(
|
|
348
|
+
"stream_mode: #{stream_mode.inspect} is deprecated, and removed in Rocket Job 8: " \
|
|
349
|
+
"#{stream_mode_replacement(stream_mode.to_s)}, see #{RocketJob::Deprecation::UPGRADE_GUIDE_V72}"
|
|
350
|
+
)
|
|
351
|
+
end
|
|
352
|
+
|
|
192
353
|
# Returns the lambda that the upload calls with each record before the first that it stores, instead of storing
|
|
193
354
|
# it, until the lambda returns true, see `Sliced::Writer::Input`: it reads the header row, when the file has one,
|
|
194
355
|
# then calls the supplied `on_first` with it. Otherwise it calls `on_first` with the first record. Returns nil
|
|
@@ -197,10 +358,14 @@ module RocketJob
|
|
|
197
358
|
# A blank line before the header row is skipped, as IOStreams skips it when it reads the header row itself, such
|
|
198
359
|
# as in :hash mode, so that the header row is not stored as a record.
|
|
199
360
|
#
|
|
200
|
-
# The header row is
|
|
201
|
-
#
|
|
202
|
-
|
|
203
|
-
|
|
361
|
+
# The header row is read whatever the mode, see #mode, since a file in a format is uploaded in lines, or in the
|
|
362
|
+
# :array `stream_mode:` of an upload, as an Array, which IOStreams also reads as a header row, see #each_record.
|
|
363
|
+
#
|
|
364
|
+
# Supply `uploaded: true` when the input of this category already holds the records of an earlier upload, which
|
|
365
|
+
# the workers read by the header row that it read, see #columns. With a schema, those columns are then kept, and
|
|
366
|
+
# the lambda raises ArgumentError for a file whose header row differs, see InputSchemaHeader.
|
|
367
|
+
def extract_header_callback(on_first, uploaded: false)
|
|
368
|
+
return schema_header_callback(on_first, uploaded) if schema.present?
|
|
204
369
|
return first_record_callback(on_first) unless tabular? && tabular.header?
|
|
205
370
|
|
|
206
371
|
lambda do |line|
|
|
@@ -235,37 +400,13 @@ module RocketJob
|
|
|
235
400
|
|
|
236
401
|
# Returns [IOStreams::Tabular] that reads the records of this category by its schema, see Category::Base#schema,
|
|
237
402
|
# and has read the header row that an upload read from the file, see #columns, so that the workers parse each line
|
|
238
|
-
# by it.
|
|
239
|
-
def schema_tabular
|
|
240
|
-
tabular = super
|
|
403
|
+
# by it. Without the supplied columns, the tabular reads the header row of a file itself.
|
|
404
|
+
def schema_tabular(columns = self.columns)
|
|
405
|
+
tabular = super()
|
|
241
406
|
tabular.read_header(columns) if columns.present? && tabular.header?
|
|
242
407
|
tabular
|
|
243
408
|
end
|
|
244
409
|
|
|
245
|
-
# Returns the lambda that reads the header row of a file with a schema, see #extract_header_callback. Its columns
|
|
246
|
-
# are matched to the fields of the schema, which raises IOStreams::Errors::InvalidHeader when they do not match,
|
|
247
|
-
# and are kept as they are in the file, see #columns. A blank row before it, which IOStreams does not read as a
|
|
248
|
-
# header row, such as a line of only commas, is skipped, as IOStreams skips it when it reads the header row itself.
|
|
249
|
-
#
|
|
250
|
-
# With a schema, `columns` hold the header row of the file uploaded, so an upload reads the header row of its own
|
|
251
|
-
# file, rather than keep the columns of an earlier one, such as those that a job scheduled again copies from its
|
|
252
|
-
# last run, see Plugins::Cron.
|
|
253
|
-
def schema_header_callback(on_first)
|
|
254
|
-
self.columns = nil
|
|
255
|
-
@tabular = nil
|
|
256
|
-
return first_record_callback(on_first) unless tabular.header?
|
|
257
|
-
|
|
258
|
-
lambda do |line|
|
|
259
|
-
tabular.read_header(line)
|
|
260
|
-
next false if tabular.header?
|
|
261
|
-
|
|
262
|
-
self.columns = tabular.header.columns
|
|
263
|
-
# Call chained on_first if present
|
|
264
|
-
on_first&.call(line)
|
|
265
|
-
true
|
|
266
|
-
end
|
|
267
|
-
end
|
|
268
|
-
|
|
269
410
|
# The schema describes the columns of the file, so the options that describe them without one cannot be set with
|
|
270
411
|
# it, see Category::Base#schema.
|
|
271
412
|
def fields_replaced_by_schema
|
|
@@ -275,10 +416,59 @@ module RocketJob
|
|
|
275
416
|
replaced
|
|
276
417
|
end
|
|
277
418
|
|
|
419
|
+
# Returns [true|false] whether the supplied `validate_rows` is true without a format, whose lines it validates.
|
|
420
|
+
def validate_rows_without_format?(validate_rows)
|
|
421
|
+
validate_rows == true && !tabular?
|
|
422
|
+
end
|
|
423
|
+
|
|
424
|
+
# Adds an error when `validate_rows` is true without a format, see #validate_rows_without_format?.
|
|
425
|
+
def validate_rows_has_format
|
|
426
|
+
errors.add(:validate_rows, VALIDATE_ROWS_NEEDS_FORMAT) if validate_rows_without_format?(validate_rows)
|
|
427
|
+
end
|
|
428
|
+
|
|
429
|
+
# Returns [true|false] whether the supplied `stream_mode:` of an upload reads a file into records rather than
|
|
430
|
+
# lines, see #each_record: without a schema, :array, whose rows IOStreams reads a header row from, and :hash
|
|
431
|
+
# without a format, which reads each line into a Hash in the format of the file name. In a format, :hash uploads
|
|
432
|
+
# the lines, whose header row is read, see #extract_header_callback, and which the workers parse, since a Hash
|
|
433
|
+
# cannot be read as a header row, nor parsed by the format.
|
|
434
|
+
def stream_mode_records?(mode)
|
|
435
|
+
schema.blank? && (mode == :array || (mode == :hash && !tabular?))
|
|
436
|
+
end
|
|
437
|
+
|
|
438
|
+
# Returns [true|false] whether an upload of the lines of a file in the supplied `stream_mode:` leaves out its
|
|
439
|
+
# DEPRECATED_UPLOAD_OPTIONS, see #each_record: in the :hash mode without a schema, when the upload reads no header
|
|
440
|
+
# row, such as for JSON, whose Hashes 7.1 read without applying them, or fixed width, which 7.1 could not read in
|
|
441
|
+
# the :hash mode without a schema. With a schema, or a header row, such as CSV, 7.1 raised for them.
|
|
442
|
+
def stream_mode_ignores_column_options?(mode)
|
|
443
|
+
mode == :hash && schema.blank? && !tabular.header?
|
|
444
|
+
end
|
|
445
|
+
|
|
446
|
+
# Returns [String] what replaces the supplied `stream_mode:` of an upload into this category, see
|
|
447
|
+
# #warn_deprecated_stream_mode.
|
|
448
|
+
def stream_mode_replacement(stream_mode)
|
|
449
|
+
if %w[hash array].include?(stream_mode) && !tabular?
|
|
450
|
+
"give the category format: :auto in place of stream_mode: :#{stream_mode}, so that the workers read each " \
|
|
451
|
+
"line into a Hash, in the format of the file name"
|
|
452
|
+
elsif stream_mode == "hash"
|
|
453
|
+
"use validate_rows: true in place of stream_mode: :hash, to parse every line of the file in the category's " \
|
|
454
|
+
"format when it is uploaded"
|
|
455
|
+
elsif stream_mode == "array"
|
|
456
|
+
"remove it, so that the workers parse the lines in the category's format, and use validate_rows: true to " \
|
|
457
|
+
"also parse every line when the file is uploaded"
|
|
458
|
+
elsif stream_mode == "line" && upload_validates_rows?
|
|
459
|
+
"use validate_rows: false in its place, since the category parses every line of a file when it is " \
|
|
460
|
+
"uploaded, which stream_mode: :line does not"
|
|
461
|
+
else
|
|
462
|
+
"remove it, since the upload is the same without it"
|
|
463
|
+
end
|
|
464
|
+
end
|
|
465
|
+
|
|
278
466
|
# The :array mode cannot be used with a schema, see #mode.
|
|
279
467
|
def schema_replaces_fields
|
|
280
468
|
super
|
|
281
|
-
|
|
469
|
+
return unless schema.present? && mode == :array
|
|
470
|
+
|
|
471
|
+
errors.add(:mode, "cannot be :array with a schema, use :line, and validate_rows: true to parse every line")
|
|
282
472
|
end
|
|
283
473
|
|
|
284
474
|
# Returns the lambda that calls the supplied `on_first` with the first record uploaded, which is not stored, see
|
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
module RocketJob
|
|
2
|
+
module Category
|
|
3
|
+
# How an input category with a schema reads the header row of a file that is uploaded into it, see
|
|
4
|
+
# Input#extract_header_callback.
|
|
5
|
+
#
|
|
6
|
+
# The workers read every record of the input by the header row that its upload read, see Input#columns. So when the
|
|
7
|
+
# input already holds the records of an earlier upload, the category keeps those columns when the upload fails,
|
|
8
|
+
# such as for a header row that does not match the schema, or a line that `validate_rows` cannot parse.
|
|
9
|
+
module InputSchemaHeader
|
|
10
|
+
# Yields the lambda that reads the header row of a file that is uploaded into this category, see
|
|
11
|
+
# Input#extract_header_callback, and returns what the block returns. Supply `uploaded: true` when the input already
|
|
12
|
+
# holds the records of an earlier upload: when the block raises, the category then keeps the columns that those
|
|
13
|
+
# records are read by. Either way its tabular is then built again from its columns, see #schema_header_callback.
|
|
14
|
+
def upload_header(on_first, uploaded: false)
|
|
15
|
+
uploaded_columns = columns
|
|
16
|
+
yield extract_header_callback(on_first, uploaded: uploaded)
|
|
17
|
+
rescue Exception
|
|
18
|
+
self.columns = uploaded_columns if uploaded
|
|
19
|
+
raise
|
|
20
|
+
ensure
|
|
21
|
+
@tabular = nil if uploaded
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
private
|
|
25
|
+
|
|
26
|
+
# Returns the lambda that reads the header row of a file with a schema. Its columns are matched to the fields of
|
|
27
|
+
# the schema, which raises IOStreams::Errors::InvalidHeader when they do not match, and are kept as they are in
|
|
28
|
+
# the file, see Input#columns. A blank row before it, which IOStreams does not read as a header row, such as a line
|
|
29
|
+
# of only commas, is skipped, as IOStreams skips it when it reads the header row itself.
|
|
30
|
+
#
|
|
31
|
+
# An upload reads the header row of its own file, rather than keep the columns of an earlier one, such as those
|
|
32
|
+
# that a job scheduled again copies from its last run, see Plugins::Cron. When the input already holds the
|
|
33
|
+
# records of an earlier upload, see `uploaded`, a tabular of its own reads the header row, and the lines of the
|
|
34
|
+
# file while it is uploaded, see Input#each_record, so that the columns stay those of the earlier upload, and the
|
|
35
|
+
# lambda raises ArgumentError when the header row differs from them, see #check_uploaded_header.
|
|
36
|
+
def schema_header_callback(on_first, uploaded)
|
|
37
|
+
uploaded_columns = columns if uploaded
|
|
38
|
+
if uploaded_columns.present?
|
|
39
|
+
@tabular = schema_tabular(nil)
|
|
40
|
+
else
|
|
41
|
+
self.columns = nil
|
|
42
|
+
@tabular = nil
|
|
43
|
+
end
|
|
44
|
+
return first_record_callback(on_first) unless tabular.header?
|
|
45
|
+
|
|
46
|
+
lambda do |line|
|
|
47
|
+
tabular.read_header(line)
|
|
48
|
+
next false if tabular.header?
|
|
49
|
+
|
|
50
|
+
if uploaded_columns.present?
|
|
51
|
+
check_uploaded_header(tabular.header.columns)
|
|
52
|
+
else
|
|
53
|
+
self.columns = tabular.header.columns
|
|
54
|
+
end
|
|
55
|
+
on_first&.call(line)
|
|
56
|
+
true
|
|
57
|
+
end
|
|
58
|
+
end
|
|
59
|
+
|
|
60
|
+
# Raises ArgumentError when the supplied header row of a file reads its columns as other fields of the schema, or
|
|
61
|
+
# in another order, than the columns that the records already in the input were uploaded with, since the workers
|
|
62
|
+
# read every record of the input by one header row, see Input#columns. A header row that the schema reads the
|
|
63
|
+
# same, such as one that differs only in the case or spaces of a column, is accepted.
|
|
64
|
+
def check_uploaded_header(header_columns)
|
|
65
|
+
return if header_fields(header_columns) == header_fields(columns)
|
|
66
|
+
|
|
67
|
+
raise(ArgumentError,
|
|
68
|
+
"Input category #{name} holds records read by the header row #{columns.inspect}, so a file whose header " \
|
|
69
|
+
"row is #{header_columns.inspect}, whose columns are other fields, cannot be uploaded into it")
|
|
70
|
+
end
|
|
71
|
+
|
|
72
|
+
# Returns [Array<String>] what the schema of the tabular reads each of the supplied columns as: the name of the
|
|
73
|
+
# field that it is, otherwise the key of a column that the schema keeps, see `extraColumns`, or nil for a blank
|
|
74
|
+
# column, or one that it skips. The nils at the end are left out, since the workers read no value from them.
|
|
75
|
+
def header_fields(header_columns)
|
|
76
|
+
file_schema = tabular.schema
|
|
77
|
+
fields = header_columns.collect do |column|
|
|
78
|
+
next if IOStreams::Utils.blank_value?(column)
|
|
79
|
+
|
|
80
|
+
field = file_schema.field_for(column)
|
|
81
|
+
next field.name if field
|
|
82
|
+
|
|
83
|
+
file_schema.key_for(column) if file_schema.keep_extra?
|
|
84
|
+
end
|
|
85
|
+
fields.pop while !fields.empty? && fields.last.nil?
|
|
86
|
+
fields
|
|
87
|
+
end
|
|
88
|
+
end
|
|
89
|
+
end
|
|
90
|
+
end
|