rocketjob 7.1.0 → 7.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -5,6 +5,9 @@ module RocketJob
5
5
  include SemanticLogger::Loggable
6
6
  include Plugins::Document
7
7
  include Category::Base
8
+ include Category::InputSchemaProposal
9
+ include Category::InputSchemaHeader
10
+ include Category::InputTextLines
8
11
 
9
12
  embedded_in :job, class_name: "RocketJob::Job", inverse_of: :input_categories
10
13
 
@@ -49,22 +52,51 @@ module RocketJob
49
52
  field :skip_unknown, type: ::Mongoid::Boolean, default: false
50
53
  validates_inclusion_of :skip_unknown, in: [true, false]
51
54
 
52
- # How `#upload` reads a file into a category with a schema, see #schema, unless the upload is given a
53
- # `stream_mode:`:
55
+ # Whether `#upload` parses each line of a file in this category's format, see #format, as it uploads it, so that a
56
+ # file with a line that cannot be read, or with a value that is not valid for its field, see #schema, uploads
57
+ # nothing. The lines are still uploaded as they are, and parsed again by the workers, see #parse.
58
+ #
59
+ # Use it when the entire file must be validated before processing is started, and upload time is not important,
60
+ # since the lines are parsed one after another while the file is uploaded.
61
+ #
62
+ # Needs a format. Only a file is validated: not a query or a range, which upload values rather than the lines of
63
+ # a file, nor the records that the block of an upload writes, see RocketJob::Batch::IO#upload.
64
+ #
65
+ # Replaces the :hash `mode`, which is deprecated, see #mode, and is the same option in Rocket Job 8.
66
+ #
67
+ # Default: nil, which validates the lines of a category with a schema in the :hash `mode`, as 7.1 did. Set it to
68
+ # false so that such a category does not validate them.
69
+ field :validate_rows, type: ::Mongoid::Boolean
70
+ validate :validate_rows_has_format
71
+
72
+ # Why `validate_rows` cannot be true without a format, see #validate_rows_without_format?.
73
+ VALIDATE_ROWS_NEEDS_FORMAT = "needs a format, whose lines it validates".freeze
74
+ private_constant :VALIDATE_ROWS_NEEDS_FORMAT
75
+
76
+ # The options of an upload, see RocketJob::Batch::IO#upload, that this category reads a file with, see
77
+ # #each_record, which describe the columns of the file, and are deprecated: the schema replaces them, see #schema.
78
+ DEPRECATED_UPLOAD_OPTIONS = %i[allowed_columns required_columns skip_unknown cleanse_header].freeze
79
+
80
+ # The modes of an input category that are deprecated, see #mode and .warn_deprecated_mode.
81
+ DEPRECATED_MODES = %w[array hash].freeze
82
+
83
+ # How `#upload` reads a file into this category, unless the upload is given a `stream_mode:`, which is deprecated:
54
84
  # :line
55
- # Uploads the lines of the file, which the workers parse by the schema, so that each value reaches `#perform` as
56
- # its type, such as a Date. This is the default, and the most performant, since the workers parse the lines.
85
+ # Uploads the lines of the file, which the workers parse, by the schema when there is one, see #schema, so that
86
+ # each value reaches `#perform` as its type, such as a Date. This is the default, and the most performant, since
87
+ # the workers parse the lines.
57
88
  # :hash
58
- # Also parses every line by the schema when it is uploaded, so that a file with a value that is not valid is not
59
- # uploaded at all. The lines are uploaded, as in :line mode. Not recommended for large files, since the lines
60
- # are parsed one after the other during the upload.
89
+ # Deprecated, use `validate_rows: true`, see #validate_rows, which Rocket Job 8 reads a saved :hash mode as.
90
+ # With a schema, see #schema, parses every line when it is uploaded, as `validate_rows: true` does. Without a
91
+ # schema it has no effect on how a file is read.
61
92
  # :array
62
- # Cannot be used with a schema.
93
+ # Deprecated, use :line, and `validate_rows: true` to also parse every line when the file is uploaded. Cannot be
94
+ # used with a schema.
63
95
  #
64
- # Without a schema the mode does not change how a file is uploaded: its lines are uploaded, unless the upload is
65
- # given `stream_mode: :array` or `stream_mode: :hash`, which parses each line into an Array or a Hash first, see
66
- # RocketJob::Batch::IO#upload. A category with a format whose files have a header row, such as CSV, cannot be
67
- # uploaded with `stream_mode: :hash` without a schema.
96
+ # Only the deprecated `stream_mode:` of an upload reads a file into records rather than lines: :hash without a
97
+ # format, and :array without a schema, see #each_record. Otherwise the lines of the file are uploaded, and its
98
+ # header row read, whatever this mode is, and the workers parse them, by the schema when there is one, so that
99
+ # each value reaches `#perform` as its type, such as a Date. The :array mode cannot be used with a schema.
68
100
  field :mode, type: ::Mongoid::StringifiedSymbol, default: :line
69
101
  validates_inclusion_of :mode, in: %i[line array hash]
70
102
 
@@ -89,6 +121,48 @@ module RocketJob
89
121
 
90
122
  validates_inclusion_of :serializer, in: %i[none compress encrypt]
91
123
 
124
+ # Reports a deprecation warning, see RocketJob::Deprecation, for the deprecated options in the supplied
125
+ # properties of an input category, see Category::Base.warn_deprecated_options, and for its :array or :hash `mode`,
126
+ # see .warn_deprecated_mode.
127
+ def self.warn_deprecated_options(properties)
128
+ super
129
+ warn_deprecated_mode(properties.to_h.stringify_keys["mode"])
130
+ end
131
+
132
+ # Reports a deprecation warning, see RocketJob::Deprecation, for the supplied mode of an input category, see
133
+ # #mode, when it is deprecated: :array, and :hash, which `validate_rows` replaces, see #validate_rows.
134
+ def self.warn_deprecated_mode(mode)
135
+ case mode.to_s
136
+ when "array"
137
+ RocketJob::Deprecation.warn(
138
+ "mode: :array is deprecated, use :line, the default, which the workers parse, and validate_rows: true to " \
139
+ "also parse every line when the file is uploaded, see #{RocketJob::Deprecation::UPGRADE_GUIDE_V72}"
140
+ )
141
+ when "hash"
142
+ RocketJob::Deprecation.warn(
143
+ "mode: :hash is deprecated, use validate_rows: true instead, which parses every line of a file in the " \
144
+ "category's format when it is uploaded, as mode: :hash did with a schema. Without a schema mode: :hash " \
145
+ "has no effect, so remove it, since Rocket Job 8 reads it as validate_rows: true, see " \
146
+ "#{RocketJob::Deprecation::UPGRADE_GUIDE_V72}"
147
+ )
148
+ end
149
+ end
150
+
151
+ # Returns [Array<String>] the names of the deprecated options in the supplied properties of an input category,
152
+ # with String keys, see Category::Base.deprecated_options, and `mode` when it is :array or :hash, see #mode.
153
+ def self.deprecated_options(properties)
154
+ names = super
155
+ names << "mode" if DEPRECATED_MODES.include?(properties["mode"].to_s)
156
+ names
157
+ end
158
+
159
+ # Returns [Array<String>] the names of the deprecated options of an input category, see
160
+ # Category::Base.deprecated_option_names, and its column options, see .deprecated_column_options. Without `mode`,
161
+ # which is only deprecated for DEPRECATED_MODES.
162
+ def self.deprecated_option_names
163
+ super + %w[allowed_columns required_columns skip_unknown header_cleanser]
164
+ end
165
+
92
166
  # Returns [Array<String>] the names of the deprecated options in the supplied properties of an input category,
93
167
  # with String keys, see Category::Base.deprecated_column_options: `allowed_columns`, `required_columns`,
94
168
  # `skip_unknown` when true, `header_cleanser` when :none, and `columns` without a schema, since with one they
@@ -157,38 +231,125 @@ module RocketJob
157
231
  # unless an encoding is set on the supplied path, which reads it instead. A schema set on the supplied path
158
232
  # is kept.
159
233
  path.schema(schema) if path.schema.nil?
234
+ elsif format == :fixed
235
+ # A fixed width line keeps its length, so that its columns stay in place: each non-printable character, such
236
+ # as NUL padding or a tab, is replaced with a space, and so is each byte of a character that is not ASCII.
237
+ # The :printable cleaner would remove them, since it ignores `replace:`.
238
+ path.encoding("UTF-8", cleaner: :replace_non_printable, replace: " ")
160
239
  elsif tabular?
161
- # Remove non-printable characters from tabular input formats.
162
- # Cannot change the length of fixed width lines.
163
- replace = format == :fixed ? " " : ""
164
- path.encoding("UTF-8", cleaner: :printable, replace: replace)
240
+ # Remove non-printable characters, and characters that are not ASCII, from the other tabular input formats.
241
+ path.encoding("UTF-8", cleaner: :printable, replace: "")
165
242
  end
166
243
  path
167
244
  end
168
245
 
169
- # Yields each record read from the supplied path for an upload, see RocketJob::Batch::IO#upload, in the supplied
170
- # mode, otherwise in :line mode, see `IOStreams::Stream#each`.
246
+ # Returns the supplied record of an upload as `#perform` receives it: parsed in this category's format, or as it is
247
+ # when the category has no format. A blank record of a tabular format is nil.
171
248
  #
172
- # With a schema it yields the lines of the file in the supplied mode, otherwise in this category's mode, see
173
- # #mode, and in :hash mode parses each line after the header row by the schema first, so that a value that is not
174
- # valid raises before any record is processed.
175
- def each_record(path, mode: nil, **args, &)
176
- return path.each(mode || :line, **args, &) if schema.blank?
177
-
178
- mode ||= self.mode
179
- if mode == :array
249
+ # Raises ArgumentError when the format needs a header row that has not been read yet.
250
+ def parse(record)
251
+ return record unless tabular?
252
+ return if record.blank?
253
+
254
+ if tabular.header?
180
255
  raise(ArgumentError,
181
- "Input category #{name} has a schema, which cannot be uploaded in :array mode, use :line or :hash")
256
+ "The tabular header columns _must_ be set before attempting to parse data that requires it.")
257
+ end
258
+
259
+ tabular.record_parse(record)
260
+ end
261
+
262
+ # Returns [Array] the first records of the supplied file, or stream, as `#perform` would receive them if the file
263
+ # were uploaded into this category, see RocketJob::Batch::IO#upload. A copy of this category reads the file, so
264
+ # nothing is stored, and this category is not changed. Raises what the upload, or a worker parsing a record,
265
+ # would raise.
266
+ def preview(stream, limit: 100)
267
+ copy = deep_dup
268
+ # A copy of a path, so that the encoding that an upload sets on it does not change the caller's.
269
+ path = copy.upload_path(stream.is_a?(IOStreams::Stream) ? stream.dup : stream)
270
+ header = copy.extract_header_callback(nil)
271
+ records = []
272
+ # Each record is parsed below, as a worker parses it, so it is not also parsed while it is read.
273
+ copy.each_record(path, validate_rows: false) do |record|
274
+ if header
275
+ header = nil if header.call(record)
276
+ next
277
+ end
278
+ records << copy.parse(record)
279
+ break if records.size >= limit
182
280
  end
183
- return path.each(:line, **args, &) if mode == :line
281
+ records
282
+ end
283
+
284
+ # Yields each record read from the supplied path for an upload, see RocketJob::Batch::IO#upload: each line, which
285
+ # is first parsed in this category's format when it validates the rows of the file, see #upload_validates_rows?,
286
+ # with the supplied `validate_rows:` of the upload, so that a line that cannot be read, or a value that is not
287
+ # valid, raises before any record is processed.
288
+ #
289
+ # The supplied mode, the deprecated `stream_mode:` of the upload, decides alone whether the lines are parsed, unless
290
+ # it is nil or false: in the :hash mode each line is parsed first, as with `validate_rows: true`. It reads the file
291
+ # into records instead of lines, as 7.0 read it, see `IOStreams::Stream#each`, only where 7.0 could upload them,
292
+ # see #stream_mode_records?.
293
+ #
294
+ # In the :hash mode, the options of the upload that describe the columns of a file, which are deprecated, are
295
+ # left out where 7.1 accepted them, see #stream_mode_ignores_column_options?. Otherwise the line reader raises
296
+ # ArgumentError for them, as 7.1 did.
297
+ #
298
+ # Without a format, each line of a text file is uploaded as UTF-8, see #each_line.
299
+ def each_record(path, mode: nil, validate_rows: nil, **args, &)
300
+ mode = mode.presence&.to_sym
301
+ return path.each(mode, **args, &) if stream_mode_records?(mode)
302
+
303
+ if schema.present? && (mode || self.mode) == :array
304
+ raise(ArgumentError, "Input category #{name} has a schema, which cannot be uploaded in :array mode, use " \
305
+ ":line, and validate_rows: true to also parse every line when the file is uploaded")
306
+ end
307
+ validate = mode ? mode == :hash : upload_validates_rows?(validate_rows)
308
+ return each_line(path, **args, &) unless validate
184
309
 
310
+ args.except!(*DEPRECATED_UPLOAD_OPTIONS) if stream_mode_ignores_column_options?(mode)
185
311
  path.each(:line, **args) do |line|
186
- # The lines before the header row, and the header row itself, are read by #extract_header_callback.
187
- tabular.record_parse(line) unless tabular.header?
312
+ # The lines before the header row, and the header row itself, are read by #extract_header_callback, and a
313
+ # blank line is not parsed, since a worker does not parse one either, see #parse.
314
+ tabular.record_parse(line) unless line.blank? || tabular.header?
188
315
  yield(line)
189
316
  end
190
317
  end
191
318
 
319
+ # Returns [true|false] whether an upload of a file into this category parses each line of it as it uploads it,
320
+ # see #validate_rows: the supplied `validate_rows:` of the upload, unless it is nil, otherwise this category's,
321
+ # unless it is nil too, otherwise whether this category has a schema and the :hash `mode`, which 7.1 parsed each
322
+ # line in. Always false without a format, which has no lines to parse.
323
+ def upload_validates_rows?(validate_rows = nil)
324
+ return false unless tabular?
325
+
326
+ validate_rows = self.validate_rows if validate_rows.nil?
327
+ validate_rows.nil? ? schema.present? && mode == :hash : validate_rows
328
+ end
329
+
330
+ # Raises ArgumentError when a file cannot be uploaded into this category with the supplied `validate_rows:` of the
331
+ # upload, see #upload_validates_rows?: when it is not true, false or nil, or when it, or else this category's
332
+ # `validate_rows`, is true without a format, which has no lines to validate.
333
+ def check_upload(validate_rows = nil)
334
+ unless [nil, true, false].include?(validate_rows)
335
+ raise(ArgumentError, "Input category #{name}: validate_rows: must be true or false, not #{validate_rows.inspect}")
336
+ end
337
+ return unless validate_rows_without_format?(validate_rows.nil? ? self.validate_rows : validate_rows)
338
+
339
+ raise(ArgumentError, "Input category #{name}: validate_rows: #{VALIDATE_ROWS_NEEDS_FORMAT}, such as :auto, which " \
340
+ "reads a file in the format of its file name")
341
+ end
342
+
343
+ # Reports a deprecation warning, see RocketJob::Deprecation, for the supplied `stream_mode:` of an upload into
344
+ # this category, also nil, see RocketJob::Batch::IO#upload, which Rocket Job 8 removes, saying what replaces it
345
+ # here: a `stream_mode:` other than nil or false decides alone whether the lines are parsed, see #each_record.
346
+ def warn_deprecated_stream_mode(stream_mode)
347
+ RocketJob::Deprecation.warn(
348
+ "stream_mode: #{stream_mode.inspect} is deprecated, and removed in Rocket Job 8: " \
349
+ "#{stream_mode_replacement(stream_mode.to_s)}, see #{RocketJob::Deprecation::UPGRADE_GUIDE_V72}"
350
+ )
351
+ end
352
+
192
353
  # Returns the lambda that the upload calls with each record before the first that it stores, instead of storing
193
354
  # it, until the lambda returns true, see `Sliced::Writer::Input`: it reads the header row, when the file has one,
194
355
  # then calls the supplied `on_first` with it. Otherwise it calls `on_first` with the first record. Returns nil
@@ -197,10 +358,14 @@ module RocketJob
197
358
  # A blank line before the header row is skipped, as IOStreams skips it when it reads the header row itself, such
198
359
  # as in :hash mode, so that the header row is not stored as a record.
199
360
  #
200
- # The header row is a line, or an Array when the upload parses each line with `stream_mode: :array`, which the
201
- # format returns as it is.
202
- def extract_header_callback(on_first)
203
- return schema_header_callback(on_first) if schema.present?
361
+ # The header row is read whatever the mode, see #mode, since a file in a format is uploaded in lines, or in the
362
+ # :array `stream_mode:` of an upload, as an Array, which IOStreams also reads as a header row, see #each_record.
363
+ #
364
+ # Supply `uploaded: true` when the input of this category already holds the records of an earlier upload, which
365
+ # the workers read by the header row that it read, see #columns. With a schema, those columns are then kept, and
366
+ # the lambda raises ArgumentError for a file whose header row differs, see InputSchemaHeader.
367
+ def extract_header_callback(on_first, uploaded: false)
368
+ return schema_header_callback(on_first, uploaded) if schema.present?
204
369
  return first_record_callback(on_first) unless tabular? && tabular.header?
205
370
 
206
371
  lambda do |line|
@@ -235,37 +400,13 @@ module RocketJob
235
400
 
236
401
  # Returns [IOStreams::Tabular] that reads the records of this category by its schema, see Category::Base#schema,
237
402
  # and has read the header row that an upload read from the file, see #columns, so that the workers parse each line
238
- # by it.
239
- def schema_tabular
240
- tabular = super
403
+ # by it. Without the supplied columns, the tabular reads the header row of a file itself.
404
+ def schema_tabular(columns = self.columns)
405
+ tabular = super()
241
406
  tabular.read_header(columns) if columns.present? && tabular.header?
242
407
  tabular
243
408
  end
244
409
 
245
- # Returns the lambda that reads the header row of a file with a schema, see #extract_header_callback. Its columns
246
- # are matched to the fields of the schema, which raises IOStreams::Errors::InvalidHeader when they do not match,
247
- # and are kept as they are in the file, see #columns. A blank row before it, which IOStreams does not read as a
248
- # header row, such as a line of only commas, is skipped, as IOStreams skips it when it reads the header row itself.
249
- #
250
- # With a schema, `columns` hold the header row of the file uploaded, so an upload reads the header row of its own
251
- # file, rather than keep the columns of an earlier one, such as those that a job scheduled again copies from its
252
- # last run, see Plugins::Cron.
253
- def schema_header_callback(on_first)
254
- self.columns = nil
255
- @tabular = nil
256
- return first_record_callback(on_first) unless tabular.header?
257
-
258
- lambda do |line|
259
- tabular.read_header(line)
260
- next false if tabular.header?
261
-
262
- self.columns = tabular.header.columns
263
- # Call chained on_first if present
264
- on_first&.call(line)
265
- true
266
- end
267
- end
268
-
269
410
  # The schema describes the columns of the file, so the options that describe them without one cannot be set with
270
411
  # it, see Category::Base#schema.
271
412
  def fields_replaced_by_schema
@@ -275,10 +416,59 @@ module RocketJob
275
416
  replaced
276
417
  end
277
418
 
419
+ # Returns [true|false] whether the supplied `validate_rows` is true without a format, whose lines it validates.
420
+ def validate_rows_without_format?(validate_rows)
421
+ validate_rows == true && !tabular?
422
+ end
423
+
424
+ # Adds an error when `validate_rows` is true without a format, see #validate_rows_without_format?.
425
+ def validate_rows_has_format
426
+ errors.add(:validate_rows, VALIDATE_ROWS_NEEDS_FORMAT) if validate_rows_without_format?(validate_rows)
427
+ end
428
+
429
+ # Returns [true|false] whether the supplied `stream_mode:` of an upload reads a file into records rather than
430
+ # lines, see #each_record: without a schema, :array, whose rows IOStreams reads a header row from, and :hash
431
+ # without a format, which reads each line into a Hash in the format of the file name. In a format, :hash uploads
432
+ # the lines, whose header row is read, see #extract_header_callback, and which the workers parse, since a Hash
433
+ # cannot be read as a header row, nor parsed by the format.
434
+ def stream_mode_records?(mode)
435
+ schema.blank? && (mode == :array || (mode == :hash && !tabular?))
436
+ end
437
+
438
+ # Returns [true|false] whether an upload of the lines of a file in the supplied `stream_mode:` leaves out its
439
+ # DEPRECATED_UPLOAD_OPTIONS, see #each_record: in the :hash mode without a schema, when the upload reads no header
440
+ # row, such as for JSON, whose Hashes 7.1 read without applying them, or fixed width, which 7.1 could not read in
441
+ # the :hash mode without a schema. With a schema, or a header row, such as CSV, 7.1 raised for them.
442
+ def stream_mode_ignores_column_options?(mode)
443
+ mode == :hash && schema.blank? && !tabular.header?
444
+ end
445
+
446
+ # Returns [String] what replaces the supplied `stream_mode:` of an upload into this category, see
447
+ # #warn_deprecated_stream_mode.
448
+ def stream_mode_replacement(stream_mode)
449
+ if %w[hash array].include?(stream_mode) && !tabular?
450
+ "give the category format: :auto in place of stream_mode: :#{stream_mode}, so that the workers read each " \
451
+ "line into a Hash, in the format of the file name"
452
+ elsif stream_mode == "hash"
453
+ "use validate_rows: true in place of stream_mode: :hash, to parse every line of the file in the category's " \
454
+ "format when it is uploaded"
455
+ elsif stream_mode == "array"
456
+ "remove it, so that the workers parse the lines in the category's format, and use validate_rows: true to " \
457
+ "also parse every line when the file is uploaded"
458
+ elsif stream_mode == "line" && upload_validates_rows?
459
+ "use validate_rows: false in its place, since the category parses every line of a file when it is " \
460
+ "uploaded, which stream_mode: :line does not"
461
+ else
462
+ "remove it, since the upload is the same without it"
463
+ end
464
+ end
465
+
278
466
  # The :array mode cannot be used with a schema, see #mode.
279
467
  def schema_replaces_fields
280
468
  super
281
- errors.add(:mode, "cannot be :array with a schema, use :line or :hash") if schema.present? && mode == :array
469
+ return unless schema.present? && mode == :array
470
+
471
+ errors.add(:mode, "cannot be :array with a schema, use :line, and validate_rows: true to parse every line")
282
472
  end
283
473
 
284
474
  # Returns the lambda that calls the supplied `on_first` with the first record uploaded, which is not stored, see
@@ -0,0 +1,90 @@
1
+ module RocketJob
2
+ module Category
3
+ # How an input category with a schema reads the header row of a file that is uploaded into it, see
4
+ # Input#extract_header_callback.
5
+ #
6
+ # The workers read every record of the input by the header row that its upload read, see Input#columns. So when the
7
+ # input already holds the records of an earlier upload, the category keeps those columns when the upload fails,
8
+ # such as for a header row that does not match the schema, or a line that `validate_rows` cannot parse.
9
+ module InputSchemaHeader
10
+ # Yields the lambda that reads the header row of a file that is uploaded into this category, see
11
+ # Input#extract_header_callback, and returns what the block returns. Supply `uploaded: true` when the input already
12
+ # holds the records of an earlier upload: when the block raises, the category then keeps the columns that those
13
+ # records are read by. Either way its tabular is then built again from its columns, see #schema_header_callback.
14
+ def upload_header(on_first, uploaded: false)
15
+ uploaded_columns = columns
16
+ yield extract_header_callback(on_first, uploaded: uploaded)
17
+ rescue Exception
18
+ self.columns = uploaded_columns if uploaded
19
+ raise
20
+ ensure
21
+ @tabular = nil if uploaded
22
+ end
23
+
24
+ private
25
+
26
+ # Returns the lambda that reads the header row of a file with a schema. Its columns are matched to the fields of
27
+ # the schema, which raises IOStreams::Errors::InvalidHeader when they do not match, and are kept as they are in
28
+ # the file, see Input#columns. A blank row before it, which IOStreams does not read as a header row, such as a line
29
+ # of only commas, is skipped, as IOStreams skips it when it reads the header row itself.
30
+ #
31
+ # An upload reads the header row of its own file, rather than keep the columns of an earlier one, such as those
32
+ # that a job scheduled again copies from its last run, see Plugins::Cron. When the input already holds the
33
+ # records of an earlier upload, see `uploaded`, a tabular of its own reads the header row, and the lines of the
34
+ # file while it is uploaded, see Input#each_record, so that the columns stay those of the earlier upload, and the
35
+ # lambda raises ArgumentError when the header row differs from them, see #check_uploaded_header.
36
+ def schema_header_callback(on_first, uploaded)
37
+ uploaded_columns = columns if uploaded
38
+ if uploaded_columns.present?
39
+ @tabular = schema_tabular(nil)
40
+ else
41
+ self.columns = nil
42
+ @tabular = nil
43
+ end
44
+ return first_record_callback(on_first) unless tabular.header?
45
+
46
+ lambda do |line|
47
+ tabular.read_header(line)
48
+ next false if tabular.header?
49
+
50
+ if uploaded_columns.present?
51
+ check_uploaded_header(tabular.header.columns)
52
+ else
53
+ self.columns = tabular.header.columns
54
+ end
55
+ on_first&.call(line)
56
+ true
57
+ end
58
+ end
59
+
60
+ # Raises ArgumentError when the supplied header row of a file reads its columns as other fields of the schema, or
61
+ # in another order, than the columns that the records already in the input were uploaded with, since the workers
62
+ # read every record of the input by one header row, see Input#columns. A header row that the schema reads the
63
+ # same, such as one that differs only in the case or spaces of a column, is accepted.
64
+ def check_uploaded_header(header_columns)
65
+ return if header_fields(header_columns) == header_fields(columns)
66
+
67
+ raise(ArgumentError,
68
+ "Input category #{name} holds records read by the header row #{columns.inspect}, so a file whose header " \
69
+ "row is #{header_columns.inspect}, whose columns are other fields, cannot be uploaded into it")
70
+ end
71
+
72
+ # Returns [Array<String>] what the schema of the tabular reads each of the supplied columns as: the name of the
73
+ # field that it is, otherwise the key of a column that the schema keeps, see `extraColumns`, or nil for a blank
74
+ # column, or one that it skips. The nils at the end are left out, since the workers read no value from them.
75
+ def header_fields(header_columns)
76
+ file_schema = tabular.schema
77
+ fields = header_columns.collect do |column|
78
+ next if IOStreams::Utils.blank_value?(column)
79
+
80
+ field = file_schema.field_for(column)
81
+ next field.name if field
82
+
83
+ file_schema.key_for(column) if file_schema.keep_extra?
84
+ end
85
+ fields.pop while !fields.empty? && fields.last.nil?
86
+ fields
87
+ end
88
+ end
89
+ end
90
+ end