iostreams 2.1.0 → 2.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (75) hide show
  1. checksums.yaml +4 -4
  2. data/docs/config.md +13 -9
  3. data/docs/copy_files.md +20 -2
  4. data/docs/extensions.md +18 -4
  5. data/docs/formats.md +644 -16
  6. data/docs/path.md +18 -2
  7. data/docs/streams.md +54 -9
  8. data/docs/tutorial.md +14 -2
  9. data/docs/upgrading.md +470 -23
  10. data/lib/io_streams/builder/deprecations.rb +250 -0
  11. data/lib/io_streams/builder.rb +256 -39
  12. data/lib/io_streams/bzip2/reader.rb +18 -2
  13. data/lib/io_streams/bzip2/writer.rb +11 -2
  14. data/lib/io_streams/deprecation/methods.rb +51 -0
  15. data/lib/io_streams/deprecation/once.rb +27 -0
  16. data/lib/io_streams/deprecation/option_names.rb +26 -0
  17. data/lib/io_streams/deprecation/words.rb +42 -0
  18. data/lib/io_streams/deprecation.rb +259 -0
  19. data/lib/io_streams/encode/reader.rb +54 -3
  20. data/lib/io_streams/encode/writer.rb +19 -2
  21. data/lib/io_streams/encode.rb +99 -0
  22. data/lib/io_streams/errors.rb +51 -0
  23. data/lib/io_streams/gzip/reader.rb +11 -0
  24. data/lib/io_streams/io_streams.rb +365 -35
  25. data/lib/io_streams/line/reader.rb +2 -1
  26. data/lib/io_streams/line/utf8_reader.rb +23 -0
  27. data/lib/io_streams/path.rb +125 -35
  28. data/lib/io_streams/paths/file.rb +72 -9
  29. data/lib/io_streams/paths/http.rb +48 -2
  30. data/lib/io_streams/paths/matcher.rb +6 -3
  31. data/lib/io_streams/paths/s3.rb +180 -9
  32. data/lib/io_streams/paths/sftp.rb +47 -3
  33. data/lib/io_streams/paths.rb +23 -0
  34. data/lib/io_streams/pgp/reader.rb +18 -0
  35. data/lib/io_streams/pgp/writer.rb +16 -0
  36. data/lib/io_streams/pgp.rb +16 -2
  37. data/lib/io_streams/reader.rb +44 -0
  38. data/lib/io_streams/record/reader.rb +59 -68
  39. data/lib/io_streams/record/writer.rb +17 -10
  40. data/lib/io_streams/row/header_row.rb +21 -0
  41. data/lib/io_streams/row/reader.rb +52 -15
  42. data/lib/io_streams/row/writer.rb +64 -13
  43. data/lib/io_streams/schema/columns.rb +456 -0
  44. data/lib/io_streams/schema/constraints.rb +99 -0
  45. data/lib/io_streams/schema/field.rb +967 -0
  46. data/lib/io_streams/schema/json_number.rb +56 -0
  47. data/lib/io_streams/schema/keys.rb +78 -0
  48. data/lib/io_streams/schema/pattern.rb +257 -0
  49. data/lib/io_streams/schema/source.rb +31 -0
  50. data/lib/io_streams/schema.rb +616 -0
  51. data/lib/io_streams/stream.rb +836 -67
  52. data/lib/io_streams/symmetric_encryption/reader.rb +8 -0
  53. data/lib/io_streams/tabular/header.rb +29 -0
  54. data/lib/io_streams/tabular/header_mapping.rb +332 -0
  55. data/lib/io_streams/tabular/mapping.rb +165 -0
  56. data/lib/io_streams/tabular/parser/array.rb +5 -0
  57. data/lib/io_streams/tabular/parser/base.rb +24 -0
  58. data/lib/io_streams/tabular/parser/fixed.rb +197 -0
  59. data/lib/io_streams/tabular/parser/fixed_fields.rb +160 -0
  60. data/lib/io_streams/tabular/parser/hash.rb +10 -0
  61. data/lib/io_streams/tabular/parser/json.rb +17 -0
  62. data/lib/io_streams/tabular/parser/json_fields.rb +132 -0
  63. data/lib/io_streams/tabular/schema_mapping.rb +361 -0
  64. data/lib/io_streams/tabular.rb +344 -30
  65. data/lib/io_streams/utils.rb +441 -6
  66. data/lib/io_streams/version.rb +1 -1
  67. data/lib/io_streams/writer.rb +27 -0
  68. data/lib/io_streams/xlsx/cells.rb +127 -0
  69. data/lib/io_streams/xlsx/reader.rb +56 -0
  70. data/lib/io_streams/xlsx/schema_reader.rb +90 -0
  71. data/lib/io_streams/xlsx/v3_writer.rb +16 -0
  72. data/lib/io_streams/zip/reader.rb +7 -0
  73. data/lib/io_streams/zip/writer.rb +24 -2
  74. data/lib/iostreams.rb +21 -17
  75. metadata +28 -2
@@ -0,0 +1,456 @@
1
+ # frozen_string_literal: true
2
+
3
+ module IOStreams
4
+ class Schema
5
+ # The columns of one file, matched to the fields of its schema: the columns of its header row, or the keys of each
6
+ # record of a format whose records supply their own keys, such as JSON. The schema finds the field of each column,
7
+ # see `IOStreams::Schema#field_for`, and checks them against its `fieldsMatch`, see
8
+ # `IOStreams::Schema#check_columns`.
9
+ #
10
+ # Converts each row and record that is read into the values of its fields, in the order of the fields, and gives the
11
+ # value of each field of each row and record that is written, so that the columns that the schema does not list are
12
+ # skipped, unless reading keeps them, see `IOStreams::Schema#keep_extra?`, after the fields. A column is matched
13
+ # once, when the header row is read or written, or once for each list of keys of the records, which are usually the
14
+ # same for every record, rather than for each value.
15
+ #
16
+ # Each reader or writer has its own, see `IOStreams::Schema#columns`, since it holds the columns of its file.
17
+ class Columns
18
+ # The most sets of keys whose plans it keeps, see #read_plan and #write_plan, so that records whose keys vary, such
19
+ # as JSON records that leave out the keys without a value, are matched once for each set of keys, while records
20
+ # whose keys are all different do not fill the memory.
21
+ PLANS = 64
22
+
23
+ # The columns kept of a file whose schema does not keep them, see #extra_columns.
24
+ NO_EXTRA = [].freeze
25
+
26
+ def initialize(schema)
27
+ @schema = schema
28
+ @keep_extra = schema.keep_extra?
29
+ # The fields whose values the rows and records read have, those that are not skipped, see
30
+ # `IOStreams::Schema::Field#skip?`, in order, and whether any is skipped.
31
+ @values = schema.fields.reject(&:skip?).freeze
32
+ @skips = @values.size != schema.fields.size
33
+ # The position and key of each column of the header row read that the schema does not list, which it keeps, in
34
+ # the order of the columns, see #extra_columns.
35
+ @extra = NO_EXTRA
36
+ # Each field of the header row read, in the order of the fields, with the position of its column and its key
37
+ # in the records read, see `IOStreams::Schema#key`, and the number of columns of the header row.
38
+ @matched = nil
39
+ @width = nil
40
+ # Whether the columns are the fields in order, for a file without a header row, see #match_fields.
41
+ @fields_in_order = false
42
+ # Each field, in order, with the position of its column in the header row written, or nil, see #match_written,
43
+ # and the number of columns of that header row.
44
+ @written = nil
45
+ @written_width = nil
46
+ # The plans of the records read, of the rows of the records read, and of the records written, by their keys, and
47
+ # the keys and plan of the last of each.
48
+ @read_plans = {}
49
+ @read_keys = nil
50
+ @read_plan = nil
51
+ @row_plans = {}
52
+ @row_keys = nil
53
+ @row_plan = nil
54
+ @write_plans = {}
55
+ @write_keys = nil
56
+ @write_plan = nil
57
+ end
58
+
59
+ # Matches the columns of a header row read to the fields. The columns without a name at the end of the header row,
60
+ # such as the empty column after a delimiter at the end of the line, which a spreadsheet exports, are not columns,
61
+ # see #named, so a value in one is a value after the last column, see #row.
62
+ #
63
+ # Returns [Array<String|Symbol>] the keys of the fields of the columns in the records read, in the order of the
64
+ # fields, see #names.
65
+ #
66
+ # Raises `IOStreams::Errors::InvalidHeader` when the columns do not match the fields, see
67
+ # `IOStreams::Schema#check_columns`.
68
+ def match(labels)
69
+ labels = named(labels)
70
+ found, unknown, positions = find_fields(labels, "column")
71
+ @schema.check_columns(found, unknown, ordered: true, noun: "column")
72
+ @matched = @schema.fields.filter_map do |field|
73
+ [field, positions[field], @schema.key(field)] if positions.key?(field) && !field.skip?
74
+ end
75
+ @extra = extra_columns(labels, positions, "column")
76
+ @width = labels.size
77
+ names
78
+ end
79
+
80
+ # Matches the columns of a file without a header row, of a format that has one, such as CSV, which are the
81
+ # fields, in order, see `IOStreams::Schema#header?`.
82
+ #
83
+ # Raises ArgumentError when the schema cannot describe such a file, see `IOStreams::Schema#check_fields_in_order`.
84
+ def match_fields
85
+ @schema.check_fields_in_order
86
+ @fields_in_order = true
87
+ match_in_order
88
+ end
89
+
90
+ # Matches the columns of the rows of a format without a header row, such as fixed width or JSON, which are the
91
+ # fields, in order, whatever `fieldsMatch` is, since it applies to the columns of the file, which the format
92
+ # matches itself: by the position of each field of a fixed width file, or by the keys of each record of JSON,
93
+ # see #read_row.
94
+ def match_in_order
95
+ @matched = @schema.fields.each_with_index.filter_map do |field, index|
96
+ [field, index, @schema.key(field)] unless field.skip?
97
+ end
98
+ @width = @schema.fields.size
99
+ end
100
+
101
+ # Returns [true|false] whether the columns of the file read are known, see #match.
102
+ def matched?
103
+ !@matched.nil?
104
+ end
105
+
106
+ # Matches the columns of a header row that is written before its rows, such as the header row read from the file
107
+ # that is copied, see `IOStreams::Row::Writer`, so that the value of each field is taken from its column of each
108
+ # row written after it. The header row of the file is still the names of the fields, so the rows are written in
109
+ # the order of the fields, with no value for a field that the header row does not have.
110
+ #
111
+ # The columns without a name at the end of the header row are not columns, as when reading, see #match.
112
+ #
113
+ # Raises `IOStreams::Errors::InvalidHeader` when a column is not a field, unless `fieldsMatch` skips it, since a
114
+ # file that is written always has every field, or when no column is a field, so that the rows after it would have
115
+ # no value, see `IOStreams::Schema#check_written_columns`.
116
+ def match_written(labels)
117
+ labels = named(labels)
118
+ found, unknown, positions = find_fields(labels, "column")
119
+ @schema.check_written_columns(found, unknown, fields_in_order: @fields_in_order)
120
+ @written = @schema.fields.map { |field| [field, field.skip? ? nil : positions[field]] }
121
+ @written_width = labels.size
122
+ end
123
+
124
+ # Returns [true|false] whether the header row of the rows written has been matched, see #match_written.
125
+ def written?
126
+ !@written.nil?
127
+ end
128
+
129
+ # Returns [Array<String|Symbol>] the keys of the fields of the columns in the records read, in the order of the
130
+ # fields, which are their names unless the schema forms them otherwise, see `IOStreams::Schema#key`, then those of
131
+ # the columns that it keeps, see #extra_columns, each a String of its own, so that a caller that changes one does
132
+ # not change the field.
133
+ def names
134
+ names = matched.map { |_field, _index, key| key.dup }
135
+ @extra.each { |extra| names << extra.last.dup }
136
+ names
137
+ end
138
+
139
+ # Returns [Array] the values of a row read, in the order of the columns, converted to the type of each field,
140
+ # in the order of the fields, without the columns that the schema does not list, unless it keeps them, see
141
+ # #extra_columns, whose values follow, as they are.
142
+ #
143
+ # Raises `IOStreams::Errors::InvalidHeader` when the row has values after its last column, see #check_extra.
144
+ def row(values)
145
+ columns = matched
146
+ check_extra(values, @width, "column")
147
+ row = columns.map { |field, index| field.cast(values[index]) }
148
+ @extra.each { |extra| row << values[extra.first] }
149
+ row
150
+ end
151
+
152
+ # Returns [Hash] the values of a row read, in the order of the columns, converted to the type of each field, by
153
+ # the key of the field, see `IOStreams::Schema#key`, in the order of the fields, then the values of the columns
154
+ # that the schema keeps, see #extra_columns, as they are.
155
+ #
156
+ # Raises `IOStreams::Errors::InvalidHeader` when the row has values after its last column, see #check_extra.
157
+ def record(values)
158
+ columns = matched
159
+ check_extra(values, @width, "column")
160
+ record = {}
161
+ columns.each { |field, index, key| record[key] = field.cast(values[index]) }
162
+ @extra.each { |index, key| record[key] = values[index] }
163
+ record
164
+ end
165
+
166
+ # Returns [Hash] a record read, whose keys are its columns, converted to the type of each field, by the key of
167
+ # the field, see `IOStreams::Schema#key`, in the order of the fields, then the values of the keys that the schema
168
+ # keeps, see #extra_columns, as they are.
169
+ #
170
+ # Raises `IOStreams::Errors::InvalidHeader` when its keys do not match the fields, see
171
+ # `IOStreams::Schema#check_columns`.
172
+ def read_hash(hash)
173
+ record = {}
174
+ read_plan(hash.keys).each { |field, key, name| record[name] = field ? field.cast(hash[key]) : hash[key] }
175
+ record
176
+ end
177
+
178
+ # Returns [Array] the value of each field, in the order of the fields, from a record read, whose keys are its
179
+ # columns, converted to the type of each field, or nil for a field that it does not have, so that the record is a
180
+ # row of the fields, see #match_in_order. Its keys are matched to the fields as for #read_hash.
181
+ #
182
+ # Raises `IOStreams::Errors::InvalidHeader` when its keys do not match the fields, see
183
+ # `IOStreams::Schema#check_columns`.
184
+ # Raises ArgumentError when the schema keeps the keys that it does not list, see #extra_columns, which the rows
185
+ # cannot hold, since each record can have other keys, which the header row of the rows does not name.
186
+ def read_row(hash)
187
+ if @keep_extra
188
+ raise(ArgumentError, "A schema with extraColumns \"keep\" cannot read the rows of a format whose records " \
189
+ "supply their own keys, such as JSON, since each record can have keys that the header row " \
190
+ "of the rows does not name. Read its records, with each(:hash)")
191
+ end
192
+
193
+ keys = hash.keys
194
+ plan = keys.eql?(@row_keys) ? @row_plan : row_plan(keys)
195
+ plan.map { |field, key| key.nil? ? nil : field.cast(hash[key]) }
196
+ end
197
+
198
+ # Returns [Array|Hash] the value of each field of the schema, in order, in the form that the format writes it, see
199
+ # `IOStreams::Schema::Field#render`, from the supplied record, whose keys are matched to the fields like columns,
200
+ # or row, whose values are the fields in order, or are in the columns of the header row written, see
201
+ # #match_written. A field that it does not supply is nil. Returns a Hash by the names of the fields when `keyed`,
202
+ # for a format whose records supply their own keys, such as JSON.
203
+ #
204
+ # Raises `IOStreams::Errors::InvalidHeader` when it has a key, or more values than there are fields or columns,
205
+ # that the schema does not skip, see `IOStreams::Schema#check_unknown`.
206
+ # Raises `IOStreams::Errors::TypeMismatch` when it is neither a Hash nor an Array.
207
+ def render(row, form, keyed)
208
+ return render_values(row, form, keyed) if row.is_a?(::Array)
209
+ unless row.is_a?(::Hash)
210
+ raise(IOStreams::Errors::TypeMismatch,
211
+ "Don't know how to write a #{row.class.name} with a schema, which writes a Hash or an Array")
212
+ end
213
+
214
+ keys = row.keys
215
+ render_record(row, form, keyed, keys.eql?(@write_keys) ? @write_plan : write_plan(keys))
216
+ end
217
+
218
+ # Returns nil, the line of a blank record or row, which a file writes as a blank line, see
219
+ # `IOStreams::Tabular#render`.
220
+ #
221
+ # Raises `IOStreams::Errors::InvalidValue` for a field that is required, since a blank record has no value for it.
222
+ def render_blank
223
+ @schema.fields.each { |field| field.cast(nil) if field.required? }
224
+ nil
225
+ end
226
+
227
+ # Matches the keys of a record read to the fields, as #read_hash does, without reading its values, such as those of
228
+ # the first record of a file whose records are handed on without being parsed, see
229
+ # `IOStreams::Tabular::SchemaMapping#check_keys`.
230
+ #
231
+ # Raises `IOStreams::Errors::InvalidHeader` when they do not match the fields, see `IOStreams::Schema#check_columns`.
232
+ def check_keys(keys)
233
+ read_plan(keys)
234
+ nil
235
+ end
236
+
237
+ private
238
+
239
+ # Returns [Array] the columns of a header row up to the last one with a name, without the columns without a name
240
+ # after it, which are blank, see `IOStreams::Utils.blank_value?`, such as the empty column after a delimiter at the end
241
+ # of each line. A column without a name between other columns is still a column, which holds a position, so that
242
+ # it is a column that the schema does not list.
243
+ def named(labels)
244
+ last = labels.rindex { |label| !IOStreams::Utils.blank_value?(label) }
245
+ last ? labels[0..last] : []
246
+ end
247
+
248
+ # Returns [Array<Array(IOStreams::Schema::Field, Integer)>] each field of the columns, in the order of the fields,
249
+ # with the position of its column, see #match.
250
+ #
251
+ # Raises `IOStreams::Errors::MissingHeader` before the columns are known.
252
+ def matched
253
+ @matched || raise(IOStreams::Errors::MissingHeader, "The columns must be known before a row can be read by them")
254
+ end
255
+
256
+ # Raises `IOStreams::Errors::InvalidHeader` when a row read or written has values after its `width`, the last column
257
+ # of its header row, or the last field of a file without one, which `last` names, unless `fieldsMatch` skips them,
258
+ # since they are not the value of any field, such as when a value read holds a delimiter that is not quoted.
259
+ #
260
+ # A value there that is blank, see `IOStreams::Utils.blank_value?`, as a column without a name after the last column
261
+ # is, see #named, is not data, such as the empty value after a delimiter at the end of a line, so it is ignored,
262
+ # both when reading and when writing, so that the rows read can be written. Any other value, such as `false`, is
263
+ # data, so it raises.
264
+ def check_extra(values, width, last)
265
+ return unless values.size > width
266
+
267
+ extra = values.drop(width).reject { |value| IOStreams::Utils.blank_value?(value) }
268
+ @schema.check_unknown(extra, noun: "value", last: last, fields_in_order: @fields_in_order)
269
+ end
270
+
271
+ # Returns [Array] the field of each label that is one, in the order of the labels, the labels that are not
272
+ # fields, and the position of the label of each field.
273
+ #
274
+ # When `by_name`, of two labels that are the same field, the one that is its name or key, see
275
+ # `IOStreams::Schema#key`, is the field, and the other is not a field, such as the key `"zip"` of a record written
276
+ # that also has the key `:zip` of the field `:zip`.
277
+ #
278
+ # Raises `IOStreams::Errors::InvalidHeader` when two labels are the same field, such as `id` and `ID`.
279
+ def find_fields(labels, noun, by_name: false)
280
+ found = []
281
+ unknown = []
282
+ positions = {}.compare_by_identity
283
+ labels.each_with_index do |label, index|
284
+ field = @schema.field_for(label, noun)
285
+ # The label before it is the name of the field, so this one is not the field.
286
+ field = nil if by_name && field && positions.key?(field) && own_label?(labels[positions[field]], field)
287
+ if field.nil?
288
+ unknown << label
289
+ elsif !positions.key?(field)
290
+ positions[field] = index
291
+ found << field
292
+ elsif by_name && own_label?(label, field)
293
+ unknown << labels[positions[field]]
294
+ positions[field] = index
295
+ else
296
+ raise(
297
+ IOStreams::Errors::InvalidHeader,
298
+ "The #{noun}s #{labels[positions[field]].inspect} and #{label.inspect} are both the field " \
299
+ "#{field.name.to_s.inspect}, so either could be meant"
300
+ )
301
+ end
302
+ end
303
+ [found, unknown, positions]
304
+ end
305
+
306
+ # Returns [Array<Array(Integer, String|Symbol)>] the position and key of each label that is not a field, in the
307
+ # order of the labels, when the schema keeps them, see `IOStreams::Schema#keep_extra?`, each by the key that it
308
+ # forms from the label, see `IOStreams::Schema#key_for`, without those without a name, which are blank, see
309
+ # `IOStreams::Utils.blank_value?`, since they cannot be a key.
310
+ #
311
+ # Raises `IOStreams::Errors::InvalidHeader` when a label kept has the key of a field, or of another label kept,
312
+ # such as `Notes` and `notes` with keyType "cleansed", since a record could not hold both.
313
+ def extra_columns(labels, positions, noun)
314
+ return NO_EXTRA unless @keep_extra
315
+
316
+ taken = positions.values.to_h { |index| [index, true] }
317
+ keys = positions.to_h { |field, index| [@schema.key(field), labels[index]] }
318
+ extra = []
319
+ labels.each_with_index do |label, index|
320
+ next if taken.key?(index) || IOStreams::Utils.blank_value?(label)
321
+
322
+ key = @schema.key_for(label)
323
+ if keys.key?(key)
324
+ raise(IOStreams::Errors::InvalidHeader,
325
+ "The #{noun}s #{keys[key].inspect} and #{label.inspect} are both the key #{key.inspect} of the records " \
326
+ "read, so either could be meant")
327
+ end
328
+
329
+ keys[key] = label
330
+ extra << [index, key]
331
+ end
332
+ extra.freeze
333
+ end
334
+
335
+ # Returns [true|false] whether the label is the name or key of the field itself, see #find_fields.
336
+ def own_label?(label, field)
337
+ label.eql?(field.name) || label.eql?(@schema.key(field))
338
+ end
339
+
340
+ # Returns [Array<Array(IOStreams::Schema::Field, Object, String|Symbol)>] each field that the keys of a record
341
+ # read supply, in the order of the fields, with the key that supplies it, and its key in the record read, see
342
+ # `IOStreams::Schema#key`, then each key that the schema keeps, see #extra_columns, without a field, worked out
343
+ # once for each list of keys. The keys are compared as a Hash compares them, so that a key that is only `==` to
344
+ # another, such as `1.0` and `1`, is not taken for it.
345
+ def read_plan(keys)
346
+ return @read_plan if keys.eql?(@read_keys)
347
+
348
+ @read_plan = @read_plans.fetch(keys) do
349
+ found, unknown, positions = find_fields(keys, "key")
350
+ @schema.check_columns(found, unknown, ordered: false, noun: "key")
351
+ @read_plans.clear if @read_plans.size >= PLANS
352
+ plan = @schema.fields.filter_map do |field|
353
+ [field, keys[positions[field]], @schema.key(field)] if positions.key?(field) && !field.skip?
354
+ end
355
+ extra_columns(keys, positions, "key").each { |index, name| plan << [nil, keys[index], name] }
356
+ @read_plans[keys] = plan
357
+ end
358
+ @read_keys = keys
359
+ @read_plan
360
+ end
361
+
362
+ # Returns [Array<Array(IOStreams::Schema::Field, Object)>] each field of the schema that is not skipped, in order,
363
+ # with the key of a record read that supplies it, see #read_plan, or nil when the record does not, worked out once
364
+ # for each list of keys. A key is never nil, since no field is the column nil, see `IOStreams::Schema#field_for`.
365
+ def row_plan(keys)
366
+ @row_plan = @row_plans.fetch(keys) do
367
+ supplied = read_plan(keys).to_h { |field, key, _name| [field, key] }
368
+ @row_plans.clear if @row_plans.size >= PLANS
369
+ @row_plans[keys] = @values.map { |field| [field, supplied[field]] }
370
+ end
371
+ @row_keys = keys
372
+ @row_plan
373
+ end
374
+
375
+ # Returns [Array<Array(IOStreams::Schema::Field, Object, Integer)>] each field of the schema, in order, with the key
376
+ # of a record written that supplies it, or nil when the record does not, and its position, worked out once for
377
+ # each list of keys, which are compared as a Hash compares them, see #read_plan. When `fieldsMatch` skips the keys
378
+ # that are not fields, of two keys that are the same field, the one that is its name supplies it, as a layout of
379
+ # the deprecated format options always took it. A field that is skipped, see `IOStreams::Schema::Field#skip?`,
380
+ # takes no value, so that it is written empty, whatever the key of its column holds.
381
+ #
382
+ # Raises `IOStreams::Errors::InvalidHeader` when a key is not a field, unless `fieldsMatch` skips it.
383
+ def write_plan(keys)
384
+ @write_plan = @write_plans.fetch(keys) do
385
+ _found, unknown, positions = find_fields(keys, "key", by_name: @schema.skip_unknown?)
386
+ @schema.check_unknown(unknown, noun: "key", fields_in_order: @fields_in_order)
387
+ @write_plans.clear if @write_plans.size >= PLANS
388
+ @write_plans[keys] = @schema.fields.each_with_index.map do |field, index|
389
+ [field, !field.skip? && positions.key?(field) ? keys[positions[field]] : nil, index]
390
+ end
391
+ end
392
+ @write_keys = keys
393
+ @write_plan
394
+ end
395
+
396
+ # Returns [Array|Hash] the rendered value of each field from the record, by the key that the plan gives for the
397
+ # field, see #render. A record of a format whose records supply their own keys, such as JSON, has no key for a
398
+ # field that is skipped, see `IOStreams::Schema::Field#skip?`, since it has no column whose place it holds.
399
+ def render_record(row, form, keyed, plan)
400
+ out = keyed ? {} : ::Array.new(plan.size)
401
+ plan.each do |field, key, index|
402
+ next if keyed && field.skip?
403
+
404
+ out[keyed ? field.name : index] = field.render(key.nil? ? nil : row[key], form)
405
+ end
406
+ out
407
+ end
408
+
409
+ # Returns [Array|Hash] the rendered value of each field from a row, whose values are the fields in order, or are
410
+ # in the columns of the header row written, see #render, once it checks its values after the last field or
411
+ # column, see #check_extra.
412
+ def render_values(row, form, keyed)
413
+ return render_columns(row, form, keyed) if @written
414
+ return render_skipping(row, form, keyed) if @skips
415
+
416
+ fields = @schema.fields
417
+ check_extra(row, fields.size, "field")
418
+ out = keyed ? {} : ::Array.new(fields.size)
419
+ fields.each_with_index { |field, index| out[keyed ? field.name : index] = field.render(row[index], form) }
420
+ out
421
+ end
422
+
423
+ # Returns [Array|Hash] the rendered value of each field from a row whose values are those of the fields that are
424
+ # not skipped, see `IOStreams::Schema::Field#skip?`, in order, as the rows read have them, with an empty value for
425
+ # each field that is skipped, or no key, see #render_record.
426
+ def render_skipping(row, form, keyed)
427
+ check_extra(row, @values.size, "field")
428
+ out = keyed ? {} : ::Array.new(@schema.fields.size)
429
+ position = 0
430
+ @schema.fields.each_with_index do |field, index|
431
+ if field.skip?
432
+ out[index] = field.render(nil, form) unless keyed
433
+ else
434
+ out[keyed ? field.name : index] = field.render(row[position], form)
435
+ position += 1
436
+ end
437
+ end
438
+ out
439
+ end
440
+
441
+ # Returns [Array|Hash] the rendered value of each field from a row whose values are in the columns of the header
442
+ # row written, see #match_written, with an empty value for each field that is skipped, or no key, see
443
+ # #render_record.
444
+ def render_columns(row, form, keyed)
445
+ check_extra(row, @written_width, "column")
446
+ out = keyed ? {} : ::Array.new(@written.size)
447
+ @written.each_with_index do |(field, position), index|
448
+ next if keyed && field.skip?
449
+
450
+ out[keyed ? field.name : index] = field.render(position.nil? ? nil : row[position], form)
451
+ end
452
+ out
453
+ end
454
+ end
455
+ end
456
+ end
@@ -0,0 +1,99 @@
1
+ # frozen_string_literal: true
2
+
3
+ module IOStreams
4
+ class Schema
5
+ # The constraints of a field, as Frictionless defines them, see https://datapackage.org/standard/table-schema/:
6
+ # the rules that its values must follow.
7
+ #
8
+ # Constraints:
9
+ # required: [true|false]
10
+ # Whether the field must have a value: a value that is nil, or empty text, raises
11
+ # `IOStreams::Errors::InvalidValue`, see #no_value, unless the field allows blank values, see `allowBlank` in
12
+ # `IOStreams::Schema::Field`. A header row or record without the field raises `IOStreams::Errors::InvalidHeader`,
13
+ # see `IOStreams::Schema#check_columns`, whether or not it allows blank values.
14
+ # Default: false
15
+ #
16
+ # The other constraints of Frictionless, such as `maxLength`, are not supported yet, so they raise ArgumentError,
17
+ # rather than appear to check values that they do not.
18
+ class Constraints
19
+ # The constraints of Frictionless that IOStreams does not support yet.
20
+ UNSUPPORTED = %w[unique minLength maxLength minimum maximum exclusiveMinimum exclusiveMaximum jsonSchema pattern
21
+ enum].freeze
22
+
23
+ # Parameters:
24
+ # constraints: [Hash|nil]
25
+ # The constraints, see above, whose keys are Strings or Symbols.
26
+ #
27
+ # name: [String|Symbol]
28
+ # The name of the field, for the messages of its errors.
29
+ #
30
+ # allow_blank: [true|false]
31
+ # Whether a required field can have no value, see `allowBlank` in `IOStreams::Schema::Field`.
32
+ # Default: false
33
+ #
34
+ # Raises ArgumentError for constraints that are not supported, or are not valid, and for `allow_blank` when the
35
+ # field is not required, whose values can be blank anyway.
36
+ def initialize(constraints, name, allow_blank: false)
37
+ @name = name
38
+ @required = check(constraints)
39
+ @allow_blank = check_allow_blank(allow_blank)
40
+ freeze
41
+ end
42
+
43
+ # Returns [true|false] whether the header row, or each record, must have the field, see `required` above, and,
44
+ # unless it allows blank values, a value, see #value_required?.
45
+ def required?
46
+ @required
47
+ end
48
+
49
+ # Returns [true|false] whether each value of the field must be present: when it is required, see `required`
50
+ # above, unless it allows blank values, so that a value of nil, or empty text, raises, see #no_value.
51
+ def value_required?
52
+ @required && !@allow_blank
53
+ end
54
+
55
+ # Returns nil, the value of a field that has none, unless each value of the field must be present, see
56
+ # #value_required?, which raises `IOStreams::Errors::InvalidValue`.
57
+ def no_value
58
+ return unless value_required?
59
+
60
+ raise(IOStreams::Errors::InvalidValue, "Field #{@name.inspect} is required, but has no value")
61
+ end
62
+
63
+ private
64
+
65
+ # Returns [true|false] whether the field is required, the only constraint that IOStreams supports yet.
66
+ def check(constraints)
67
+ return false if constraints.nil?
68
+ unless constraints.is_a?(Hash)
69
+ raise(ArgumentError, "Field #{@name.inspect}: constraints must be a Hash: #{constraints.inspect}")
70
+ end
71
+
72
+ constraints = constraints.transform_keys(&:to_s)
73
+ other = (constraints.keys - ["required"]).first
74
+ raise(ArgumentError, "Field #{@name.inspect}: #{problem(other)}") if other
75
+
76
+ required = constraints["required"]
77
+ return false if required.nil?
78
+ return required if [true, false].include?(required)
79
+
80
+ raise(ArgumentError, "Field #{@name.inspect}: constraints required must be true or false: #{required.inspect}")
81
+ end
82
+
83
+ # Returns [true|false] whether the field allows blank values, which only a required field can.
84
+ def check_allow_blank(allow_blank)
85
+ return allow_blank if !allow_blank || @required
86
+
87
+ raise(ArgumentError, "Field #{@name.inspect}: allowBlank only applies to a field whose constraints are required, " \
88
+ "since the values of any other field can be blank")
89
+ end
90
+
91
+ # Returns [String] why a constraint other than `required` raises.
92
+ def problem(constraint)
93
+ return "the constraint #{constraint.inspect} is not supported" if UNSUPPORTED.include?(constraint)
94
+
95
+ "unknown constraint #{constraint.inspect}"
96
+ end
97
+ end
98
+ end
99
+ end