iostreams 2.1.0 → 2.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (75) hide show
  1. checksums.yaml +4 -4
  2. data/docs/config.md +13 -9
  3. data/docs/copy_files.md +20 -2
  4. data/docs/extensions.md +18 -4
  5. data/docs/formats.md +644 -16
  6. data/docs/path.md +18 -2
  7. data/docs/streams.md +54 -9
  8. data/docs/tutorial.md +14 -2
  9. data/docs/upgrading.md +470 -23
  10. data/lib/io_streams/builder/deprecations.rb +250 -0
  11. data/lib/io_streams/builder.rb +256 -39
  12. data/lib/io_streams/bzip2/reader.rb +18 -2
  13. data/lib/io_streams/bzip2/writer.rb +11 -2
  14. data/lib/io_streams/deprecation/methods.rb +51 -0
  15. data/lib/io_streams/deprecation/once.rb +27 -0
  16. data/lib/io_streams/deprecation/option_names.rb +26 -0
  17. data/lib/io_streams/deprecation/words.rb +42 -0
  18. data/lib/io_streams/deprecation.rb +259 -0
  19. data/lib/io_streams/encode/reader.rb +54 -3
  20. data/lib/io_streams/encode/writer.rb +19 -2
  21. data/lib/io_streams/encode.rb +99 -0
  22. data/lib/io_streams/errors.rb +51 -0
  23. data/lib/io_streams/gzip/reader.rb +11 -0
  24. data/lib/io_streams/io_streams.rb +365 -35
  25. data/lib/io_streams/line/reader.rb +2 -1
  26. data/lib/io_streams/line/utf8_reader.rb +23 -0
  27. data/lib/io_streams/path.rb +125 -35
  28. data/lib/io_streams/paths/file.rb +72 -9
  29. data/lib/io_streams/paths/http.rb +48 -2
  30. data/lib/io_streams/paths/matcher.rb +6 -3
  31. data/lib/io_streams/paths/s3.rb +180 -9
  32. data/lib/io_streams/paths/sftp.rb +47 -3
  33. data/lib/io_streams/paths.rb +23 -0
  34. data/lib/io_streams/pgp/reader.rb +18 -0
  35. data/lib/io_streams/pgp/writer.rb +16 -0
  36. data/lib/io_streams/pgp.rb +16 -2
  37. data/lib/io_streams/reader.rb +44 -0
  38. data/lib/io_streams/record/reader.rb +59 -68
  39. data/lib/io_streams/record/writer.rb +17 -10
  40. data/lib/io_streams/row/header_row.rb +21 -0
  41. data/lib/io_streams/row/reader.rb +52 -15
  42. data/lib/io_streams/row/writer.rb +64 -13
  43. data/lib/io_streams/schema/columns.rb +456 -0
  44. data/lib/io_streams/schema/constraints.rb +99 -0
  45. data/lib/io_streams/schema/field.rb +967 -0
  46. data/lib/io_streams/schema/json_number.rb +56 -0
  47. data/lib/io_streams/schema/keys.rb +78 -0
  48. data/lib/io_streams/schema/pattern.rb +257 -0
  49. data/lib/io_streams/schema/source.rb +31 -0
  50. data/lib/io_streams/schema.rb +616 -0
  51. data/lib/io_streams/stream.rb +836 -67
  52. data/lib/io_streams/symmetric_encryption/reader.rb +8 -0
  53. data/lib/io_streams/tabular/header.rb +29 -0
  54. data/lib/io_streams/tabular/header_mapping.rb +332 -0
  55. data/lib/io_streams/tabular/mapping.rb +165 -0
  56. data/lib/io_streams/tabular/parser/array.rb +5 -0
  57. data/lib/io_streams/tabular/parser/base.rb +24 -0
  58. data/lib/io_streams/tabular/parser/fixed.rb +197 -0
  59. data/lib/io_streams/tabular/parser/fixed_fields.rb +160 -0
  60. data/lib/io_streams/tabular/parser/hash.rb +10 -0
  61. data/lib/io_streams/tabular/parser/json.rb +17 -0
  62. data/lib/io_streams/tabular/parser/json_fields.rb +132 -0
  63. data/lib/io_streams/tabular/schema_mapping.rb +361 -0
  64. data/lib/io_streams/tabular.rb +344 -30
  65. data/lib/io_streams/utils.rb +441 -6
  66. data/lib/io_streams/version.rb +1 -1
  67. data/lib/io_streams/writer.rb +27 -0
  68. data/lib/io_streams/xlsx/cells.rb +127 -0
  69. data/lib/io_streams/xlsx/reader.rb +56 -0
  70. data/lib/io_streams/xlsx/schema_reader.rb +90 -0
  71. data/lib/io_streams/xlsx/v3_writer.rb +16 -0
  72. data/lib/io_streams/zip/reader.rb +7 -0
  73. data/lib/io_streams/zip/writer.rb +24 -2
  74. data/lib/iostreams.rb +21 -17
  75. metadata +28 -2
@@ -1,7 +1,83 @@
1
1
  module IOStreams
2
2
  class Stream
3
- attr_reader :io_stream
4
- attr_writer :builder
3
+ # How to set each setting of the file on the path instead, which the readers and writers of rows and records accept
4
+ # for each call, see #path_setting_names, and v3.0 only accepts on the path. A file name supplied to a call only
5
+ # chooses its format, when the path has none, and not its streams, as #file_name does, so the format replaces it.
6
+ PATH_SETTINGS = {
7
+ format: "set it on the path with #format instead",
8
+ file_name: "where it only chooses the format when the path has none, set the format on the path with " \
9
+ "#format instead",
10
+ original_file_name: "where it only chooses the format when the path has none, set the format on the path with " \
11
+ "#format instead",
12
+ default_format: "set the format on the path with #format instead, when the path has none, such as " \
13
+ "path.format(:json) unless path.format",
14
+ format_options: "set a schema on the path with #schema instead, which gives the width of each field of a " \
15
+ "fixed width file, such as the schema that IOStreams::Schema.from_layout returns for the " \
16
+ "layout, converted once and saved, since v3.0 removes from_layout"
17
+ }.freeze
18
+ private_constant :PATH_SETTINGS
19
+
20
+ FORMAT_OPTIONS_DEPRECATION = "#format_options is deprecated, use #schema instead, which gives the width of each " \
21
+ "field of a fixed width file".freeze
22
+ private_constant :FORMAT_OPTIONS_DEPRECATION
23
+
24
+ # The modes that a copy converts the data with, see #copy_from. Without a mode it copies the bytes.
25
+ COPY_MODES = %i[line array hash].freeze
26
+ private_constant :COPY_MODES
27
+
28
+ # The number of bytes that a copy without a mode reads at a time, where it does not copy with `IO.copy_stream`, see
29
+ # #copy_data, as many as CRuby's `IO.copy_stream` reads.
30
+ COPY_SIZE = 16_384
31
+ private_constant :COPY_SIZE
32
+
33
+ # Whether `IO.copy_stream` writes the text that a source that is not an IO reads in the encoding that the source
34
+ # gives it, as CRuby's does, so that a target that converts text, such as with #encoding, converts it, rather than
35
+ # writing it as binary, as JRuby's does, see https://github.com/jruby/jruby/issues/9881, which such a target takes as
36
+ # text already in its own encoding. Found by what it does, so that #copy_from uses it again once JRuby keeps the
37
+ # encoding.
38
+ COPY_STREAM_KEEPS_ENCODING =
39
+ begin
40
+ text = "\u00E9"
41
+ source = Object.new
42
+ source.define_singleton_method(:read) do |_length = nil, buffer = nil|
43
+ data = text
44
+ text = nil
45
+ buffer && data ? buffer.replace(data) : data
46
+ end
47
+ encoding = nil
48
+ target = Object.new
49
+ target.define_singleton_method(:write) do |data|
50
+ encoding = data.encoding
51
+ data.bytesize
52
+ end
53
+ IO.copy_stream(source, target)
54
+ encoding == Encoding::UTF_8
55
+ end
56
+
57
+ IO_STREAM_DEPRECATION = "#io_stream is deprecated, since v3.0 makes it private, keep the IO supplied to " \
58
+ "IOStreams.stream instead".freeze
59
+ private_constant :IO_STREAM_DEPRECATION
60
+
61
+ # The deprecation warning of a method of the stream that a reader block receives, which the stream that v3.0
62
+ # supplies in its place does not answer, after the name of the method, with the name of the stream, and the methods
63
+ # that the stream of v3.0 answers, see #v3_deprecation.
64
+ V3_READER_DEPRECATION = "is deprecated on %<name>s that #reader yields, since in v3.0 it is a stream that only " \
65
+ "reads, with %<methods>s: read it with those, open the reader again to read it from the " \
66
+ "start, or copy the data into a local file first, such as with IOStreams.temp_file and " \
67
+ "#copy_to, where it must be a File".freeze
68
+ private_constant :V3_READER_DEPRECATION
69
+
70
+ # The deprecation warning of a method of the stream that a writer block receives, see V3_READER_DEPRECATION.
71
+ V3_WRITER_DEPRECATION = "is deprecated on %<name>s that #writer yields, since in v3.0 it is a stream that only " \
72
+ "writes, with %<methods>s: write it with those, or write a local file first, such as with " \
73
+ "IOStreams.temp_file, and copy it with #copy_to, where it must be a File".freeze
74
+ private_constant :V3_WRITER_DEPRECATION
75
+
76
+ # The deprecation warning of a method of the stream that a reader or writer block receives, which v3.0 removes, and
77
+ # which the stream names the replacement of, after the name of the method, with the name of the stream, the method
78
+ # that yields it, and the replacement, see #v3_deprecation.
79
+ V3_REMOVED_DEPRECATION = "is deprecated on %s that %s yields, since v3.0 removes it, %s".freeze
80
+ private_constant :V3_REMOVED_DEPRECATION
5
81
 
6
82
  def initialize(io_stream)
7
83
  raise(ArgumentError, "io_stream cannot be nil") if io_stream.nil?
@@ -11,6 +87,20 @@ module IOStreams
11
87
  @builder = nil
12
88
  end
13
89
 
90
+ # A copy has its own streams, options and encoding, so that changing them does not change the original, see
91
+ # `IOStreams::Builder#initialize_copy`.
92
+ def initialize_copy(source)
93
+ super
94
+ @builder = @builder&.dup
95
+ end
96
+
97
+ # Deprecated: returns the IO, or other object, supplied to `IOStreams.stream`, which the caller already holds, or nil
98
+ # for a path. v3.0 makes it private. Reports a deprecation warning, see `IOStreams::Deprecation`.
99
+ def io_stream
100
+ Deprecation.warn(IO_STREAM_DEPRECATION)
101
+ supplied_io
102
+ end
103
+
14
104
  # Ignore the filename and use only the supplied streams.
15
105
  #
16
106
  # See #option to set an option for one of the streams included based on the file name extensions.
@@ -19,6 +109,7 @@ module IOStreams
19
109
  #
20
110
  # IOStreams.path("tempfile2527").stream(:zip).stream(:pgp, passphrase: "receiver_passphrase").read
21
111
  def stream(stream, **)
112
+ warn_change("#stream")
22
113
  builder.stream(stream, **)
23
114
  self
24
115
  end
@@ -34,6 +125,7 @@ module IOStreams
34
125
  #
35
126
  # IOStreams.path(output_file_name).option(:pgp, passphrase: "receiver_passphrase").read
36
127
  def option(stream, **)
128
+ warn_change("#option")
37
129
  builder.option(stream, **)
38
130
  self
39
131
  end
@@ -41,10 +133,75 @@ module IOStreams
41
133
  # Adds the options for the specified stream as an option,
42
134
  # but if streams have already been added it is instead added as a stream.
43
135
  def option_or_stream(stream, **)
136
+ warn_change("#option_or_stream")
44
137
  builder.option_or_stream(stream, **)
45
138
  self
46
139
  end
47
140
 
141
+ # Read and write the data as it is stored: no streams are applied, neither those that the file name implies
142
+ # nor any set with #stream or #option, and no encoding set with #encoding.
143
+ # Setting a stream, option or encoding afterwards raises ArgumentError.
144
+ #
145
+ # Examples:
146
+ #
147
+ # # Checksum a compressed file as it is stored.
148
+ # Digest::SHA256.hexdigest(IOStreams.path("data.csv.gz").raw.read)
149
+ #
150
+ # # Write data that is already compressed.
151
+ # IOStreams.path("data.csv.gz").raw.write(gzipped_data)
152
+ #
153
+ # To copy a file as it is stored, such as downloading a zip file without unzipping it, use
154
+ # `copy_to(target, convert: false)`, since the target applies its own streams. To read a file that is not named
155
+ # for its format, such as a CSV file called `data.txt`, set the file name that names its format instead, for
156
+ # example `file_name("data.csv")`.
157
+ #
158
+ # Replaces `stream(:none)`, which is deprecated: it still works, and reports a deprecation warning, see
159
+ # `IOStreams::Deprecation`.
160
+ def raw
161
+ warn_change("#raw")
162
+ builder.raw
163
+ self
164
+ end
165
+
166
+ # Returns [true|false] whether the data is read and written as it is stored, see #raw.
167
+ def raw?
168
+ builder.raw?
169
+ end
170
+
171
+ # Set the character encoding of the text that the application reads or writes, which the encode stream
172
+ # converts, and its other options, `replace:` and `cleaner:`, see `IOStreams::Encode::Reader`. Unlike the streams
173
+ # in the file name, the encoding stays with the path, whichever streams it has, and works for a stream without a
174
+ # file name too. Each call merges its options with those already set.
175
+ #
176
+ # Raises ArgumentError when it is set, rather than when the data is read or written, for an option that the
177
+ # encode stream does not take, or a value that it does not accept, such as an encoding that Ruby does not know, see
178
+ # `IOStreams::Encode.check`.
179
+ #
180
+ # Examples:
181
+ #
182
+ # # Read binary lines from a compressed file, which is still decompressed.
183
+ # IOStreams.path("data.csv.gz").encoding("BINARY").each(:line) { |line| p line }
184
+ #
185
+ # # Read UTF-8 text, removing the characters that are not printable.
186
+ # IOStreams.path("export.csv").encoding("UTF-8", cleaner: :printable).each(:hash) { |hash| p hash }
187
+ #
188
+ # Replaces `option(:encode, ...)`, with the same options and the same results. Also replaces
189
+ # `stream(:encode, ...)`, which, like any other stream set with #stream, stops the streams being taken from the
190
+ # file name, which #encoding keeps: so to read or write the data as it is stored, as
191
+ # `stream(:encode, encoding: "BINARY")` does for a file name such as `data.csv.gz`, use #raw instead, and to read a
192
+ # file that is not named for its format, #file_name, then #encoding. Both are deprecated: they still work, and
193
+ # report a deprecation warning, see `IOStreams::Deprecation`.
194
+ #
195
+ # Without any arguments, returns [String|Encoding|nil] the encoding that was set, or nil when none was, like
196
+ # `IO#encoding`.
197
+ def encoding(encoding = nil, **options)
198
+ return builder.setting(:encode)&.[](:encoding) if encoding.nil? && options.empty?
199
+
200
+ warn_change("#encoding")
201
+ builder.encoding(encoding, **options)
202
+ self
203
+ end
204
+
48
205
  # Return the options already set for either a stream or option.
49
206
  def setting(stream)
50
207
  builder.setting(stream)
@@ -60,6 +217,7 @@ module IOStreams
60
217
  # If the stream pipeline has not yet been built it will be built from the file_name if present.
61
218
  # Note: Any options must be set _before_ calling this method.
62
219
  def remove_from_pipeline(stream_name)
220
+ warn_change("#remove_from_pipeline")
63
221
  builder.remove_from_pipeline(stream_name)
64
222
  self
65
223
  end
@@ -88,10 +246,13 @@ module IOStreams
88
246
  #
89
247
  # Notes:
90
248
  # - Newlines embedded within quoted fields are kept within the same line when
91
- # 1. The resolved tabular format quotes its fields (e.g. CSV, whether detected from a
92
- # `.csv` file name or set explicitly via `.format(:csv)`)
93
- # 2. Or the `embedded_within` argument is supplied (e.g. `embedded_within: '"'`)
94
- # - Pass `embedded_within: nil` to disable quote-aware line joining for a quoted format.
249
+ # 1. The tabular format quotes its fields, such as CSV, whether set explicitly via `.format(:csv)`,
250
+ # or detected from a `.csv` file name.
251
+ # 2. Or, in the `:line` mode, the `embedded_within` argument is supplied (e.g. `embedded_within: '"'`).
252
+ # - Pass `embedded_within: nil` to read the physical lines of a quoted format in the `:line` mode.
253
+ # Supplying `embedded_within` in the `:array` and `:hash` modes is deprecated, since the format splits their
254
+ # lines: set the format with #format instead. It still applies, and reports a deprecation warning, see
255
+ # `IOStreams::Deprecation`.
95
256
  def each(mode = :line, **args, &block)
96
257
  raise(ArgumentError, "Invalid mode: #{mode.inspect}") if mode == :stream
97
258
 
@@ -105,10 +266,22 @@ module IOStreams
105
266
  end
106
267
 
107
268
  # Returns a Reader for reading a file / stream
269
+ #
270
+ # Supplying options to the `:stream` mode, which ignores them, is deprecated, and reports a deprecation warning,
271
+ # see `IOStreams::Deprecation`. So is supplying the settings of the file, such as `format:`, to the `:array` and
272
+ # `:hash` modes, which are set on the path instead, such as with #format.
273
+ #
274
+ # The stream that the block of the `:stream` mode receives reports a deprecation warning for each of its methods that
275
+ # the application calls, and that the stream that v3.0 supplies in its place does not answer, such as `#rewind` of the
276
+ # `Zlib::GzipReader` of a `.gz` file, when deprecation warnings are on as it is opened, see #v3_deprecation.
108
277
  def reader(mode = :stream, **args, &)
109
278
  case mode
110
279
  when :stream
111
- stream_reader(&)
280
+ warn_ignored_options(args.keys, "in the :stream mode")
281
+ deprecation = v3_deprecation(:reader) if block_given?
282
+ return stream_reader(&) unless deprecation
283
+
284
+ stream_reader { |io| yield(Deprecation::Methods.deprecate(io, *deprecation)) }
112
285
  when :line
113
286
  line_reader(**args, &)
114
287
  when :array
@@ -127,14 +300,30 @@ module IOStreams
127
300
  # this method will load the entire file into memory.
128
301
  # - Recommend using instead `#reader` to read a block into memory at a time.
129
302
  def read(*args)
130
- reader { |stream| stream.read(*args) }
303
+ internal_reader { |stream| stream.read(*args) }
131
304
  end
132
305
 
133
306
  # Returns a Writer for writing to a file / stream
307
+ #
308
+ # Supplying options to the `:stream` mode, which ignores them, is deprecated, as are the settings of the file,
309
+ # such as `format:`, for the `:array` and `:hash` modes, see #reader, and the options that only apply when
310
+ # reading, such as `allowed_columns:`, which a writer ignores. So are the options supplied when the stream is a
311
+ # line, row or record writer that is already open, such as the one that `writer(:line)` yields, handed to
312
+ # `IOStreams.stream`, which writes with its own settings, and ignores them, and the streams, encoding, format and
313
+ # schema set on such a stream, see #supplied_writer. Each reports a deprecation warning, see
314
+ # `IOStreams::Deprecation`.
315
+ #
316
+ # The stream that the block of the `:stream` mode receives reports a deprecation warning for each of its methods that
317
+ # the application calls, and that the stream that v3.0 supplies in its place does not answer, such as `#fileno` of
318
+ # the IO of gpg's input when writing PGP, when deprecation warnings are on as it is opened, see #v3_deprecation.
134
319
  def writer(mode = :stream, **args, &)
135
320
  case mode
136
321
  when :stream
137
- stream_writer(&)
322
+ warn_ignored_options(args.keys, "in the :stream mode")
323
+ deprecation = v3_deprecation(:writer) if block_given?
324
+ return stream_writer(&) unless deprecation
325
+
326
+ stream_writer { |io| yield(Deprecation::Methods.deprecate(io, *deprecation)) }
138
327
  when :line
139
328
  line_writer(**args, &)
140
329
  when :array
@@ -153,7 +342,7 @@ module IOStreams
153
342
  # since all the data for the file needs to be resident in memory before writing.
154
343
  # - Recommend using instead `#writer` to write a block of memory at a time.
155
344
  def write(data)
156
- writer { |stream| stream.write(data) }
345
+ internal_writer { |stream| stream.write(data) }
157
346
  end
158
347
 
159
348
  # Copy from another stream, path, file_name or IO instance.
@@ -168,34 +357,65 @@ module IOStreams
168
357
  #
169
358
  # :mode [:line, :array, :hash]
170
359
  # When convert is `true` then use this mode to convert the contents of the file.
360
+ # Any other mode raises ArgumentError, before the source or the target is opened.
171
361
  #
172
362
  # Examples:
173
363
  #
174
364
  # # Copy and convert streams based on file extensions
175
- # IOStreams.path("target_file.json").copy_from("source_file_name.csv.gz")
365
+ # IOStreams.path("target_file.csv").copy_from("source_file_name.csv.gz")
366
+ #
367
+ # # Copy the rows of a file into a file of another format, converting them
368
+ # IOStreams.path("target_file.psv").copy_from("source_file_name.csv.gz", mode: :array)
369
+ #
370
+ # # Copy the records of a file into JSON, whose rows are records
371
+ # IOStreams.path("target_file.json").copy_from("source_file_name.csv.gz", mode: :hash)
176
372
  #
177
373
  # # Copy "as-is" without any automated stream conversions
178
- # IOStreams.path("target_file.json").copy_from("source_file_name.csv.gz", convert: false)
374
+ # IOStreams.path("target_file.csv.gz").copy_from("source_file_name.csv.gz", convert: false)
179
375
  #
180
376
  # # Advanced copy with custom stream conversions on source and target.
181
- # source = IOStreams.path("source_file").stream(:encode, encoding: "BINARY")
377
+ # source = IOStreams.path("source_file").encoding("BINARY")
182
378
  # IOStreams.path("target_file.pgp").option(:pgp, passphrase: "hello").copy_from(source)
379
+ #
380
+ # Notes:
381
+ # - When the source or the target has a schema, see #schema, a copy of rows or records hands the target the header
382
+ # row of the source as the header row of the rows after it, which a target with a schema matches to its fields,
383
+ # and skips the blank rows, see `IOStreams::Row::Reader#copy_to`. Otherwise each row or record read is written,
384
+ # as before.
385
+ # - The source is opened before the target, so that the target is not changed when the source cannot be read, for
386
+ # example when it does not exist. When the copy fails part way, the target discards its incomplete write, as a
387
+ # path does whenever a write fails.
388
+ #
389
+ # Supplying options to a copy without a `mode:`, which ignores them, is deprecated, as is supplying `mode:` or
390
+ # other options to a copy with `convert: false`. So is a copy without a `mode:` from a source in another tabular
391
+ # format than this stream, such as CSV into JSON, whose text it copies as it is, since v3.0 converts its rows, see
392
+ # #warn_copy_format. Each reports a deprecation warning, see `IOStreams::Deprecation`.
183
393
  def copy_from(source, convert: true, mode: nil, **args)
184
394
  if convert
395
+ validate_copy_mode!(mode)
185
396
  stream = IOStreams.new(source)
186
397
  if mode
187
- writer(mode, **args) do |target|
188
- stream.each(mode) { |row| target << row }
398
+ stream.reader(mode) do |rows|
399
+ writer(mode, **args) do |target|
400
+ if mode == :line || (schema.nil? && stream.schema.nil?)
401
+ rows.each { |row| target << row }
402
+ else
403
+ rows.copy_to(target)
404
+ end
405
+ end
189
406
  end
190
407
  else
191
- writer(**args) do |target|
192
- stream.reader { |src| IO.copy_stream(src, target) }
408
+ warn_ignored_options(args.keys, "when copying without a `mode:`")
409
+ warn_copy_format(stream)
410
+ stream.internal_reader do |src|
411
+ internal_writer { |target| copy_data(src, target) }
193
412
  end
194
413
  end
195
414
  else
415
+ warn_unconverted_copy_options(mode, args)
196
416
  stream = source.is_a?(Stream) ? source.dup : IOStreams.new(source)
197
- dup.stream(:none).writer do |target|
198
- stream.stream(:none).reader { |src| IO.copy_stream(src, target) }
417
+ stream.no_streams.internal_reader do |src|
418
+ dup.no_streams.internal_writer { |target| IO.copy_stream(src, target) }
199
419
  end
200
420
  end
201
421
  end
@@ -210,6 +430,7 @@ module IOStreams
210
430
  if file_name == :none
211
431
  builder.file_name
212
432
  else
433
+ warn_change("#file_name")
213
434
  builder.file_name = file_name
214
435
  self
215
436
  end
@@ -217,14 +438,20 @@ module IOStreams
217
438
 
218
439
  # Set the original file_name
219
440
  def file_name=(file_name)
441
+ warn_change("#file_name=")
220
442
  builder.file_name = file_name
221
443
  end
222
444
 
223
- # Set/get the tabular format_options
445
+ # Set/get the tabular format
446
+ #
447
+ # A format other than `:csv` set on a spreadsheet, such as `format(:psv)` on an `.xlsx` path, is deprecated, since
448
+ # its rows are read as CSV, and reports a deprecation warning when its lines, rows or records are read, see
449
+ # `IOStreams::Builder#warn_text_format`.
224
450
  def format(format = :none)
225
451
  if format == :none
226
452
  builder.format
227
453
  else
454
+ warn_change("#format")
228
455
  builder.format = format
229
456
  self
230
457
  end
@@ -232,11 +459,17 @@ module IOStreams
232
459
 
233
460
  # Set the tabular format
234
461
  def format=(format)
462
+ warn_change("#format=")
235
463
  builder.format = format
236
464
  end
237
465
 
238
- # Set/get the tabular format options
466
+ # Deprecated: use #schema, which gives the width of each field of a fixed width file, see docs/formats.md, and
467
+ # `IOStreams::Schema.from_layout` for the schema of a layout. Reports a deprecation warning, see
468
+ # `IOStreams::Deprecation`.
469
+ #
470
+ # Set/get the tabular format options: the layout of a fixed width file.
239
471
  def format_options(format_options = :none)
472
+ Deprecation.warn(FORMAT_OPTIONS_DEPRECATION)
240
473
  if format_options == :none
241
474
  builder.format_options
242
475
  else
@@ -245,11 +478,62 @@ module IOStreams
245
478
  end
246
479
  end
247
480
 
481
+ # Deprecated: use #schema=, see #format_options.
482
+ #
248
483
  # Set the tabular format_options
249
484
  def format_options=(format_options)
485
+ Deprecation.warn(FORMAT_OPTIONS_DEPRECATION)
250
486
  builder.format_options = format_options
251
487
  end
252
488
 
489
+ # Set/get the schema of the rows and records of this file: the name and type of each field, so that reading
490
+ # rows and records returns each value as its type, such as an Integer or a Date, and writing them writes each
491
+ # value in the text form of its type. It applies to every tabular format, and gives the position of each field of
492
+ # a fixed width file. See docs/formats.md, and `IOStreams::Schema` for its properties.
493
+ #
494
+ # Parameters:
495
+ # schema: [Hash|String|IOStreams::Path]
496
+ # A Frictionless Table Schema as a Hash, which is checked now, or the path or url of a JSON file that holds
497
+ # one, in any store, which is read the first time that a reader or writer needs it.
498
+ # nil removes the schema.
499
+ #
500
+ # Returns [Hash|String|IOStreams::Path] the schema as it was set, a Hash or String as a frozen copy, so that
501
+ # changing the object that was supplied afterwards does not change it, or [nil] when none was set.
502
+ #
503
+ # Notes:
504
+ # - Without a schema, rows and records are read and written as before.
505
+ # - A schema cannot be supplied to each call of #each, #reader or #writer, nor with the layout of a fixed width
506
+ # file in #format_options, which raise ArgumentError.
507
+ # - With a schema, the text of the file is read as UTF-8, unless an encoding is set with
508
+ # #encoding, and text that is not valid UTF-8 raises `IOStreams::Errors::InvalidEncoding`.
509
+ #
510
+ # Examples:
511
+ #
512
+ # # people.csv has other columns too, such as "name", which the schema skips.
513
+ # path = IOStreams.path("people.csv").schema(fieldsMatch: "partial", fields: [{name: "age", type: "integer"}])
514
+ # path.each(:hash) do |person|
515
+ # person["age"] # => 42
516
+ # end
517
+ #
518
+ # path = IOStreams.path("s3://bucket/people.txt").format(:fixed).schema("s3://bucket/schemas/people.json")
519
+ # path.each(:hash) { |person| p person }
520
+ def schema(schema = :none)
521
+ # Compared by identity, since a path compares itself to another path, see `IOStreams::Path#==`.
522
+ return builder.schema if schema.equal?(:none)
523
+
524
+ warn_change("#schema")
525
+ builder.schema = schema
526
+ self
527
+ end
528
+
529
+ # Set the schema, see #schema.
530
+ #
531
+ # Raises ArgumentError when the schema is not a Hash, a String or a path, or is a Hash that is not a valid schema.
532
+ def schema=(schema)
533
+ warn_change("#schema=")
534
+ builder.schema = schema
535
+ end
536
+
253
537
  # Returns [String] the last component of this path.
254
538
  # Returns `nil` if no `file_name` was set.
255
539
  #
@@ -265,7 +549,7 @@ module IOStreams
265
549
  file_name = builder.file_name
266
550
  return unless file_name
267
551
 
268
- suffix.nil? ? ::File.basename(file_name) : ::File.basename(file_name, suffix)
552
+ Utils::PathName.basename(file_name, suffix)
269
553
  end
270
554
 
271
555
  # Returns [String] the directory for this file.
@@ -281,7 +565,7 @@ module IOStreams
281
565
  # IOStreams.path(".profile").dirname #=> "."
282
566
  def dirname
283
567
  file_name = builder.file_name
284
- ::File.dirname(file_name) if file_name
568
+ Utils::PathName.dirname(file_name) if file_name
285
569
  end
286
570
 
287
571
  # Returns [String] the extension for this file including the last period.
@@ -301,7 +585,7 @@ module IOStreams
301
585
  # IOStreams.path(".profile.sh").extname #=> ".sh"
302
586
  def extname
303
587
  file_name = builder.file_name
304
- ::File.extname(file_name) if file_name
588
+ Utils::PathName.extname(file_name) if file_name
305
589
  end
306
590
 
307
591
  # Returns [String] the extension for this file _without_ the last period.
@@ -323,24 +607,189 @@ module IOStreams
323
607
  extname&.sub(/^\./, "")
324
608
  end
325
609
 
610
+ # Deprecated: it takes an internal object. To start again without the streams and options set with #stream or
611
+ # #option, create the path again with `IOStreams.path`. Reports a deprecation warning, see
612
+ # `IOStreams::Deprecation`.
613
+ def builder=(builder)
614
+ Deprecation.warn("#builder= is deprecated, since it takes an internal object, create the path again with " \
615
+ "IOStreams.path to start without the streams and options set with #stream or #option")
616
+ @builder = builder
617
+ end
618
+
619
+ protected
620
+
621
+ # Applies no streams, as `stream(:none)` does, for IOStreams itself, which never uses its own deprecated methods.
622
+ # `temp_file: true` for the path of `IOStreams.temp_file`, see `IOStreams::Builder#no_streams`.
623
+ def no_streams(temp_file: false)
624
+ builder.no_streams(temp_file: temp_file)
625
+ self
626
+ end
627
+
628
+ # Starts again without the streams and options set with #stream or #option, for IOStreams itself, see #builder=.
629
+ def reset_builder
630
+ @builder = nil
631
+ self
632
+ end
633
+
634
+ # Yields the stream that the block of the :stream mode of #reader receives, for IOStreams itself, such as #read or a
635
+ # copy, which calls its methods for itself, so that it reports no deprecation warning for them, see #v3_deprecation.
636
+ # It checks first that the stream may be read, as a path's #reader does, see #authorize!.
637
+ def internal_reader(&)
638
+ authorize!
639
+ stream_reader(&)
640
+ end
641
+
642
+ # Yields the stream that the block of the :stream mode of #writer receives, for IOStreams itself, see
643
+ # #internal_reader.
644
+ def internal_writer(&)
645
+ authorize!
646
+ stream_writer(&)
647
+ end
648
+
649
+ # Checks that the stream may be read or written, before #internal_reader or #internal_writer opens it: a path that
650
+ # is not within the allowed paths raises, see `IOStreams::Path#authorize!`, while a stream has nothing to check.
651
+ # They are defined here, rather than for a path, so that any stream may ask another for them, such as a copy from
652
+ # a path to a stream.
653
+ def authorize!
654
+ end
655
+
656
+ # Options supplied to a copy with `convert: false`, which ignores them, are deprecated, including `mode:`.
657
+ def warn_unconverted_copy_options(mode, args)
658
+ names = args.keys
659
+ names.unshift(:mode) unless mode.nil?
660
+ warn_ignored_options(names, "for a copy with convert: false")
661
+ end
662
+
663
+ # Returns [Symbol|nil] the tabular format of the data that a copy without a mode reads from or writes to this
664
+ # stream, when IOStreams v3.0 converts the rows of such a copy between it and another format, otherwise nil, see
665
+ # `IOStreams::Builder#copy_format`, which the stream at the other end of a copy asks for, see #warn_copy_format.
666
+ def copy_format
667
+ builder.copy_format
668
+ end
669
+
326
670
  private
327
671
 
672
+ # Copies the data that the supplied source reads to the supplied target, as `IO.copy_stream` does, keeping the
673
+ # encoding of the text that the source reads, so that a target that converts text converts it, see
674
+ # COPY_STREAM_KEEPS_ENCODING. It reads without a buffer, as JRuby's `IO.copy_stream` does, since not every source
675
+ # takes one, such as JRuby's `Zlib::GzipReader`, see https://github.com/jruby/jruby/issues/9884.
676
+ #
677
+ # Returns [Integer] the number of bytes copied, which the target says that it wrote, as for `IO.copy_stream`.
678
+ def copy_data(source, target)
679
+ return IO.copy_stream(source, target) if COPY_STREAM_KEEPS_ENCODING
680
+
681
+ copied = 0
682
+ while (data = source.read(COPY_SIZE))
683
+ copied += target.write(data)
684
+ end
685
+ copied
686
+ end
687
+
328
688
  def builder
329
689
  @builder ||= IOStreams::Builder.new
330
690
  end
331
691
 
692
+ # Returns the IO, or other object, supplied to `IOStreams.stream`, or nil for a path, for IOStreams itself, see
693
+ # #io_stream.
694
+ def supplied_io
695
+ @io_stream
696
+ end
697
+
698
+ # Called with the name of each method that changes this stream in place, such as "#stream", before it changes it,
699
+ # so that a subclass can report a deprecation warning when changing it is deprecated, see
700
+ # `IOStreams::Path#warn_change`. A stream reports none.
701
+ def warn_change(_method_name)
702
+ nil
703
+ end
704
+
705
+ # Returns [Array(Array<Symbol>, String)|nil] the methods that the stream that the block of the :stream mode of the
706
+ # supplied type receives answers in v3.0, and the deprecation warning of each of its other methods that the
707
+ # application calls, after the name of the method, which names the methods that it answers, when the stream that it
708
+ # receives in v2.2 answers others, see `IOStreams::Builder#v3_stream`, otherwise nil, so that #reader and #writer
709
+ # extend the stream with them, see `IOStreams::Deprecation::Methods`, and otherwise hand the block on unchanged.
710
+ # Known before the streams are opened, since the streams that v3.0 replaces are those of IOStreams itself, each of
711
+ # which yields one stream.
712
+ #
713
+ # Nil while deprecation warnings are off, see `Warning[:deprecated]`, so that the stream is then handed on as it is,
714
+ # without any of this work.
715
+ #
716
+ # A stream that names what replaces some of its methods, such as the encode stream, whose #encoding the #encoding of
717
+ # the path replaces, has a deprecation warning for each of them, see `IOStreams::Deprecation::Methods.deprecate`.
718
+ def v3_deprecation(type)
719
+ return unless Warning[:deprecated]
720
+
721
+ reading = type == :reader
722
+ methods, name, advice = builder.v3_stream(type, reading ? v3_source : v3_target) { reading && v3_reads_file? }
723
+ return unless methods
724
+
725
+ template = reading ? V3_READER_DEPRECATION : V3_WRITER_DEPRECATION
726
+ # Kernel.format, since #format is the tabular format of this stream.
727
+ message = Kernel.format(template, name: name, methods: method_list(methods))
728
+ return [methods, message] unless advice
729
+
730
+ messages = advice.transform_values { |use| Kernel.format(V3_REMOVED_DEPRECATION, name, "##{type}", use) }
731
+ [methods, messages.merge(nil => message)]
732
+ end
733
+
734
+ # Returns [String] the names of the methods, as a deprecation warning lists them, such as "#read, #gets and #eof?".
735
+ def method_list(methods)
736
+ names = methods.map { |name| "##{name}" }
737
+ names.size > 2 ? "#{names[0..-2].join(', ')} and #{names.last}" : names.join(" and ")
738
+ end
739
+
740
+ # Returns [Array(Array<Symbol>, String)|nil] the methods that the IO that the streams of this stream read in v3.0
741
+ # answers, and its name for a deprecation warning, when it is not the IO that they read in v2.2, see
742
+ # `IOStreams::Builder#v3_stream`, otherwise nil, as for the IO supplied to `IOStreams.stream`, which v3.0 reads too.
743
+ def v3_source
744
+ nil
745
+ end
746
+
747
+ # Returns [Array(Array<Symbol>, String)|nil] the methods that the IO that the streams of this stream write in v3.0
748
+ # answers, and its name for a deprecation warning, when it is not the IO that they write in v2.2, see #v3_source,
749
+ # otherwise nil, as for the IO supplied to `IOStreams.stream`, which v3.0 writes too.
750
+ def v3_target
751
+ nil
752
+ end
753
+
754
+ # Returns [true|false] whether the streams of this stream read a regular local file in v3.0: the IO supplied to
755
+ # `IOStreams.stream`, when it is an open File, or the File of a Tempfile, which gpg reads itself in v3.0, see
756
+ # `IOStreams::Pgp::Reader.v3_stream_methods`. Only asked by a stream that needs to know, since it checks the file,
757
+ # see `IOStreams::Builder#v3_stream`.
758
+ def v3_reads_file?
759
+ io = supplied_io
760
+ io = io.__getobj__ if defined?(::Delegator) && io.is_a?(::Delegator)
761
+ io.is_a?(::File) && !io.closed? && io.stat.file?
762
+ rescue IOError, SystemCallError
763
+ false
764
+ end
765
+
766
+ # Raises ArgumentError for a mode that a copy does not convert with, see COPY_MODES, before the source or the target
767
+ # is opened.
768
+ def validate_copy_mode!(mode)
769
+ return if mode.nil? || COPY_MODES.include?(mode)
770
+
771
+ raise(ArgumentError, "Invalid mode: #{mode.inspect}. A copy converts lines, rows or records with mode: " \
772
+ "#{COPY_MODES.map(&:inspect).join(', ')}, or copies the data, as bytes, without a mode")
773
+ end
774
+
332
775
  def stream_reader(&)
333
- builder.reader(io_stream, &)
776
+ builder.reader(supplied_io, &)
334
777
  end
335
778
 
779
+ # Yields the reader of the lines of this stream, which the readers of its rows and records read too.
780
+ #
781
+ # A format set with #format that is not that of the text of its streams, such as `:psv` for a spreadsheet, whose
782
+ # rows are read as CSV, reports a deprecation warning, see `IOStreams::Builder#warn_text_format`.
336
783
  def line_reader(embedded_within: :auto, **args)
784
+ builder.warn_text_format
337
785
  # `:auto` defers the decision to the resolved tabular format (e.g. CSV quotes with `"`),
338
786
  # while distinguishing "not supplied" from an explicit value such as `nil` (disable) or
339
787
  # `'"'` (force). Centralizing this in the builder keeps all format-based decisions there.
340
788
  embedded_within = builder.quote_character if embedded_within == :auto
789
+ reader_class = builder.utf8_text? ? IOStreams::Line::Utf8Reader : IOStreams::Line::Reader
341
790
 
342
791
  stream_reader do |io|
343
- yield IOStreams::Line::Reader.new(
792
+ yield reader_class.new(
344
793
  io,
345
794
  embedded_within: embedded_within,
346
795
  **args
@@ -349,71 +798,391 @@ module IOStreams
349
798
  end
350
799
 
351
800
  # Iterate over a file / stream returning each line as an array, one at a time.
801
+ #
802
+ # Supplying `embedded_within:` is deprecated, since the format of the file decides where its lines are split, and
803
+ # reports a deprecation warning, see `IOStreams::Deprecation`. So are the rows of a file whose rows are its records,
804
+ # such as JSON without a schema, see #warn_record_rows.
352
805
  def row_reader(delimiter: nil, embedded_within: :auto, **args)
353
- line_reader(delimiter: delimiter, embedded_within: embedded_within) do |io|
354
- yield IOStreams::Row::Reader.new(
355
- io,
356
- original_file_name: builder.file_name,
357
- format: builder.format,
358
- format_options: builder.format_options,
359
- **args
360
- )
806
+ rows_reader(IOStreams::Row::Reader, delimiter, embedded_within, args) do |reader|
807
+ warn_record_rows(reader)
808
+ yield reader
361
809
  end
362
810
  end
363
811
 
364
812
  # Iterate over a file / stream returning each line as a hash, one at a time.
365
- def record_reader(delimiter: nil, embedded_within: :auto, **args)
813
+ #
814
+ # Supplying `embedded_within:` is deprecated, see #row_reader.
815
+ #
816
+ # Records read from a header row whose columns are the same key, such as `a,a,b`, are deprecated, and report a
817
+ # deprecation warning, while deprecation warnings are on, see `IOStreams::Record::Reader#warn_repeated_key`.
818
+ def record_reader(delimiter: nil, embedded_within: :auto, **args, &)
819
+ rows_reader(IOStreams::Record::Reader, delimiter, embedded_within, args, report_repeated_key: Deprecation.on?, &)
820
+ end
821
+
822
+ # Yields the reader of the rows or records of this stream, an instance of the supplied class, with the options of
823
+ # the call.
824
+ #
825
+ # With a schema, an option that the call cannot supply raises before a deprecation warning is reported for any of
826
+ # its other options, so that a call that fails reports only its failure, see #schema_tabular. Then each deprecated
827
+ # option of the call reports a deprecation warning, see #warn_read_options.
828
+ #
829
+ # The supplied reporting options, such as `report_repeated_key:`, are passed to the reader.
830
+ def rows_reader(klass, delimiter, embedded_within, args, **reporting)
831
+ tabular = schema_tabular(args)
832
+ warn_read_options(embedded_within, args)
833
+ warn_text_file_format(args)
834
+ warn_cell_rows(delimiter)
366
835
  line_reader(delimiter: delimiter, embedded_within: embedded_within) do |io|
367
- yield IOStreams::Record::Reader.new(
368
- io,
369
- original_file_name: builder.file_name,
370
- format: builder.format,
371
- format_options: builder.format_options,
372
- **args
373
- )
836
+ yield klass.new(io, **tabular_options(tabular, args), **reporting)
837
+ end
838
+ end
839
+
840
+ # Reports a deprecation warning for the deprecated options of a call that reads rows or records: the settings of the
841
+ # file, see #warn_path_settings, `embedded_within:`, and the column options, which
842
+ # `IOStreams::Tabular.deprecated_read_option_names` names.
843
+ def warn_read_options(embedded_within, args)
844
+ warn_path_settings(args)
845
+ warn_embedded_within(embedded_within)
846
+ warn_column_options(IOStreams::Tabular.deprecated_read_option_names(args))
847
+ end
848
+
849
+ # Reports a deprecation warning for the deprecated options of a call that writes rows or records: the settings of
850
+ # the file, see #warn_path_settings, the options that only apply when reading, see #warn_read_only_options, and the
851
+ # column options, which `IOStreams::Tabular.deprecated_write_option_names` names.
852
+ def warn_write_options(args)
853
+ warn_path_settings(args)
854
+ warn_read_only_options(args)
855
+ warn_column_options(IOStreams::Tabular.deprecated_write_option_names(args))
856
+ end
857
+
858
+ # The settings of the file supplied to a call, see #path_setting_names, are deprecated, even when they are nil,
859
+ # since v3.0 raises ArgumentError for each of them. Each one reports how to set it on the path instead, see
860
+ # PATH_SETTINGS, apart from `format_options:` with a schema, which raises for a layout, see #schema_tabular, so that
861
+ # the one that it accepts, nil or empty, has no effect, and is reported with the advice to remove it.
862
+ #
863
+ # When the streams read the rows in a format of their own, such as those of a spreadsheet as CSV, see
864
+ # `IOStreams::Builder#text_format`, the settings that choose the format, see #format_setting_names, report that
865
+ # format as the one to set on the path, since any other is deprecated there too, see
866
+ # `IOStreams::Builder#warn_text_format`. It is only looked for while deprecation warnings are on, see
867
+ # `IOStreams::Deprecation.on?`, since it builds the pipeline.
868
+ def warn_path_settings(args)
869
+ names = args.keys & path_setting_names
870
+ format_names = format_setting_names
871
+ text_format = builder.text_format if Deprecation.on? && names.intersect?(format_names)
872
+ names.each do |name|
873
+ advice =
874
+ if name == :format_options && !builder.schema.nil?
875
+ "and has no effect with a schema, so remove it"
876
+ elsif text_format && format_names.include?(name)
877
+ "set format(#{text_format.inspect}) on the path instead, since the streams read the rows as " \
878
+ "#{text_format.inspect}, such as those of a spreadsheet"
879
+ else
880
+ PATH_SETTINGS.fetch(name)
881
+ end
882
+ Deprecation.warn("#{name}: is deprecated for each call, #{advice}")
883
+ end
884
+ end
885
+
886
+ # Returns [Array<Symbol>] the names of the options of a call that describe the file: those of the tabular, see
887
+ # `IOStreams::Tabular::FORMAT_OPTION_NAMES`, other than `schema:`, which raises, see #reject_schema_option!, and
888
+ # `original_file_name:`, with which the readers and writers of rows and records take the file name of the path, see
889
+ # #tabular_options, which the option of the call replaces.
890
+ def path_setting_names
891
+ [*(IOStreams::Tabular::FORMAT_OPTION_NAMES - %i[schema]), :original_file_name]
892
+ end
893
+
894
+ # Returns [Array<Symbol>] the names of the options of a call that choose the format of the file, in place of the
895
+ # format of its path: those that describe the file, see #path_setting_names, other than `format_options:`, which
896
+ # describes the fields of a fixed width file.
897
+ def format_setting_names
898
+ path_setting_names - %i[format_options]
899
+ end
900
+
901
+ # The options that only apply when reading a header, see `IOStreams::Tabular::Header::READ_OPTION_NAMES`, are
902
+ # deprecated when writing, which ignores them, even when they are nil, since v3.0 raises ArgumentError for them.
903
+ def warn_read_only_options(args)
904
+ names = args.keys & IOStreams::Tabular::Header::READ_OPTION_NAMES
905
+ return if names.empty?
906
+
907
+ words = Deprecation::Words.new(names)
908
+ advice =
909
+ if builder.schema.nil?
910
+ "describe the columns that are written with #schema instead"
911
+ else
912
+ "so remove #{words.them}"
913
+ end
914
+ Deprecation.warn("#{words.keywords} #{words.are} deprecated when writing, since " \
915
+ "#{words.one? ? 'it only applies' : 'they only apply'} when reading and #{words.are} ignored, " \
916
+ "#{advice}")
917
+ end
918
+
919
+ # The column options of each call are deprecated, since a schema describes the columns of the file on its path, see
920
+ # #schema. With a schema, which raises for those that describe the columns, see #schema_tabular, the others that it
921
+ # accepts have no effect, so they are reported with the advice to remove them.
922
+ def warn_column_options(names)
923
+ return if names.empty?
924
+
925
+ words = Deprecation::Words.new(names)
926
+ advice =
927
+ if builder.schema.nil?
928
+ "describe the columns of the file with #schema instead"
929
+ else
930
+ "and #{words.one? ? 'has' : 'have'} no effect with a schema, which describes the columns of the file, so " \
931
+ "remove #{words.them}"
932
+ end
933
+ Deprecation.warn("#{words.keywords} #{words.are} deprecated, #{advice}")
934
+ end
935
+
936
+ # Options that are ignored, such as those supplied to the `:stream` mode, are deprecated.
937
+ def warn_ignored_options(names, where)
938
+ return if names.empty?
939
+
940
+ words = Deprecation::Words.new(names)
941
+ Deprecation.warn("#{words.keywords} #{words.are} deprecated #{where}, which ignores #{words.them}, so remove " \
942
+ "#{words.them}")
943
+ end
944
+
945
+ # A copy without a `mode:` from a source in another tabular format than this stream, see #copy_format, such as CSV
946
+ # into JSON or PSV, is deprecated, since IOStreams v3.0 converts the rows of the source into the format of this
947
+ # stream, as the `:array` mode does, rather than copy its text as it is, as this version still does. The warning
948
+ # names the mode that converts them in both versions, see `IOStreams::Tabular.conversion_mode`, and `:line`, which
949
+ # copies the text as it is in both.
950
+ #
951
+ # A source or target whose format is not known, such as an IO without a file name or a `.txt` file, one read or
952
+ # written as it is stored, see #raw, and a source and target of the same format, such as a spreadsheet, whose text is
953
+ # CSV, and a CSV file, are copied as they are in both versions, so they are not reported. None is looked for while
954
+ # deprecation warnings are off, see `IOStreams::Deprecation.on?`, since finding the format of a spreadsheet builds
955
+ # its pipeline.
956
+ def warn_copy_format(source)
957
+ return unless Deprecation.on?
958
+
959
+ from = source.copy_format
960
+ to = copy_format
961
+ return if from.nil? || to.nil? || from == to
962
+
963
+ mode = IOStreams::Tabular.conversion_mode(from, to)
964
+ Deprecation.warn("a copy without mode: from #{from.inspect} into #{to.inspect} is deprecated, since v3.0 " \
965
+ "converts the rows of the source into #{to.inspect}, as mode: :array does, rather than copy its " \
966
+ "text as it is: supply mode: #{mode.inspect} to convert its #{mode == :hash ? 'records' : 'rows'}, " \
967
+ "or mode: :line to copy its text as it is in both versions")
968
+ end
969
+
970
+ # The rows and records of a file named `.text` without a format, such as `ids.text`, are deprecated, since IOStreams
971
+ # v3.0 reads and writes them in its `:text` format, as the text of each line, see
972
+ # `IOStreams::Builder#text_file_format`, while this version reads them as CSV, or in the format of an extension
973
+ # before `.text`, which `#format` sets on both. A call that supplies a setting of the file that chooses the format,
974
+ # see #format_setting_names, such as `format:` or a file name that names it, which are deprecated themselves, see
975
+ # #warn_path_settings, is not reported.
976
+ def warn_text_file_format(args)
977
+ return if args.keys.intersect?(format_setting_names)
978
+
979
+ format = builder.text_file_format
980
+ return unless format
981
+
982
+ Deprecation.warn("the rows and records of a file named .text without #format are deprecated, since v3.0 reads " \
983
+ "and writes them in its :text format, as the text of each line, rather than as " \
984
+ "#{format.to_s.upcase}: set format(:#{format}) on the path")
985
+ end
986
+
987
+ # The rows of a file whose rows are its records, such as JSON without a schema, see
988
+ # `IOStreams::Row::Reader#record_rows?`, are deprecated, since IOStreams v3.0 reads them as Arrays, after a header
989
+ # row of the keys of the first record, while this version reads each record as a Hash, as `each(:hash)` does on both.
990
+ def warn_record_rows(reader)
991
+ return unless reader.record_rows?
992
+
993
+ Deprecation.warn("each(:array) and reader(:array) of a file whose rows are records, such as JSON without a " \
994
+ "schema, are deprecated, since v3.0 yields each row as an Array, after a header row of the " \
995
+ "keys of the first record, rather than each record as a Hash: read the records with " \
996
+ "each(:hash), or set a schema")
997
+ end
998
+
999
+ # Reports what v3.0 changes about reading the rows and records of a spreadsheet, whose streams read its rows in a
1000
+ # format of their own, see `IOStreams::Builder#text_format`, since v3.0 reads them as the values of its cells, as
1001
+ # Excel typed them, rather than as lines of CSV text:
1002
+ # - Reading them without a type for each column, see `IOStreams::Schema#types_every_column?`, which yields the text of
1003
+ # each cell, such as `"1001.0"`, where v3.0 yields its value, such as the Float `1001.0`. A field with a type
1004
+ # converts both to the same value, such as `1001` for an integer.
1005
+ # - `delimiter:`, which v3.0 raises ArgumentError for, since they are not lines, as it does for an encoding set with
1006
+ # #encoding, see `IOStreams::Builder#warn_cells_encoding`.
1007
+ #
1008
+ # None is looked for while deprecation warnings are off, see `IOStreams::Deprecation.on?`, since finding the format
1009
+ # of the text builds the pipeline.
1010
+ def warn_cell_rows(delimiter)
1011
+ return unless Deprecation.on? && builder.text_format
1012
+
1013
+ unless delimiter.nil?
1014
+ Deprecation.warn("delimiter: is deprecated for the rows and records of a spreadsheet, which are its cells " \
1015
+ "rather than lines of text, since v3.0 raises ArgumentError for it, so remove it; each(:line) " \
1016
+ "still takes it")
374
1017
  end
1018
+ builder.warn_cells_encoding
1019
+ return if builder.tabular_schema&.types_every_column?
1020
+
1021
+ Deprecation.warn("the rows and records of a spreadsheet read without a type for each column are deprecated, " \
1022
+ "since v3.0 reads the value of each cell as Excel typed it, such as the Float 1001.0, a Date or " \
1023
+ "true, rather than its text, such as \"1001.0\": give each column that is read a type with " \
1024
+ "#schema, such as {name: \"id\", type: \"integer\"}, or {name: \"zip\", type: \"string\"} for " \
1025
+ "its text")
1026
+ end
1027
+
1028
+ def warn_embedded_within(embedded_within)
1029
+ return if embedded_within == :auto
1030
+
1031
+ Deprecation.warn("embedded_within: is deprecated when reading rows and records, since the format of the file " \
1032
+ "decides where their lines are split, set it with #format instead, such as format(:psv) " \
1033
+ "for values that are not quoted")
1034
+ end
1035
+
1036
+ # Returns [IOStreams::Tabular|nil] the tabular that reads or writes the rows or records of this stream with the
1037
+ # options of the call, when it has a schema, see #schema, which is created before the file is opened, so that its
1038
+ # schema file is read, and an option that the schema does not accept raises, before the file is opened, or a writer
1039
+ # replaces it. Returns nil without a schema, see #tabular_options.
1040
+ #
1041
+ # Like the readers and writers of rows and records without a schema, see #tabular_options, it takes the format from
1042
+ # the file name of the path, or from the `original_file_name:` of the call, which replaces it, or from the
1043
+ # `file_name:` of the call, which replaces both.
1044
+ #
1045
+ # Its text is taken as UTF-8, see `IOStreams::Builder#utf8_text?`, unless the application chose its encoding, or read
1046
+ # it as it is stored, when it is `decoded:`, so that the text read is taken as it is, and binary text written that
1047
+ # is not valid UTF-8 is left for the encoding. The lines that it reads are UTF-8 already, see #line_reader, so it
1048
+ # only checks them.
1049
+ #
1050
+ # Raises ArgumentError for `schema:`, which is set on the path.
1051
+ def schema_tabular(args)
1052
+ schema = builder.tabular_schema
1053
+ return unless schema
1054
+
1055
+ reject_schema_option!(args)
1056
+ IOStreams::Tabular.new(
1057
+ file_name: args.fetch(:original_file_name, builder.file_name),
1058
+ format: builder.format,
1059
+ format_options: builder.format_options,
1060
+ schema: schema,
1061
+ decoded: !builder.utf8_text?,
1062
+ **args.except(:original_file_name)
1063
+ )
1064
+ end
1065
+
1066
+ # Returns [Hash] the options of the reader or writer of the rows or records of this stream: the tabular that reads or
1067
+ # writes them with a schema, see #schema_tabular, otherwise the tabular format of this stream and the options of
1068
+ # the call, from which it creates its own, as before.
1069
+ #
1070
+ # Raises ArgumentError for `schema:`, which is set on the path.
1071
+ def tabular_options(tabular, args)
1072
+ return {tabular: tabular} if tabular
1073
+
1074
+ reject_schema_option!(args)
1075
+ {original_file_name: builder.file_name, format: builder.format, format_options: builder.format_options, **args}
1076
+ end
1077
+
1078
+ # Raises ArgumentError for `schema:` supplied to a call, since the schema describes the file, so it is set on the
1079
+ # path, see #schema, as is the format.
1080
+ def reject_schema_option!(args)
1081
+ return unless args.key?(:schema)
1082
+
1083
+ raise(ArgumentError, ":schema cannot be supplied to each call, set it on the path instead, for example: " \
1084
+ "path.schema(\"people.json\").each(:hash)")
375
1085
  end
376
1086
 
377
1087
  def stream_writer(&)
378
- builder.writer(io_stream, &)
1088
+ builder.writer(supplied_io, &)
379
1089
  end
380
1090
 
381
1091
  def line_writer(**args, &block)
382
- return block.call(io_stream) if io_stream.is_a?(IOStreams::Line::Writer)
1092
+ return supplied_writer(IOStreams::Line::Writer, args, &block) if supplied_io.is_a?(IOStreams::Line::Writer)
383
1093
 
384
- writer do |io|
1094
+ stream_writer do |io|
385
1095
  IOStreams::Line::Writer.stream(io, **args, &block)
386
1096
  end
387
1097
  end
388
1098
 
389
- def row_writer(delimiter: $/, **args, &block)
390
- return block.call(io_stream) if io_stream.is_a?(IOStreams::Row::Writer)
1099
+ # Yields the writer of the rows of this stream, see #rows_writer.
1100
+ #
1101
+ # Rows written to a format whose rows are records, such as JSON, under a header row whose columns are the same key
1102
+ # are deprecated, and report a deprecation warning, while deprecation warnings are on, see
1103
+ # `IOStreams::Row::Writer#warn_repeated_key`.
1104
+ def row_writer(**args, &)
1105
+ rows_writer(IOStreams::Row::Writer, args, report_repeated_key: Deprecation.on?, &)
1106
+ end
391
1107
 
392
- line_writer(delimiter: delimiter) do |io|
393
- IOStreams::Row::Writer.stream(
394
- io,
395
- original_file_name: builder.file_name,
396
- format: builder.format,
397
- format_options: builder.format_options,
398
- **args,
399
- &block
400
- )
1108
+ def record_writer(**args, &)
1109
+ rows_writer(IOStreams::Record::Writer, args, &)
1110
+ end
1111
+
1112
+ # Yields the writer of the rows or records of this stream, an instance of the supplied class, with the options of
1113
+ # the call, which writes its lines with the `delimiter:` of the call, or that of the line writer by default, see
1114
+ # `IOStreams::Line::Writer`.
1115
+ #
1116
+ # As with a reader, see #rows_reader, with a schema an option that the call cannot supply raises before a deprecation
1117
+ # warning is reported for any of its other options, see #warn_write_options.
1118
+ def rows_writer(klass, args, **reporting, &block)
1119
+ return supplied_writer(klass, args, &block) if supplied_io.is_a?(klass)
1120
+
1121
+ options = args.except(:delimiter)
1122
+ tabular = schema_tabular(options)
1123
+ warn_write_options(options)
1124
+ warn_text_file_format(options)
1125
+ rows_line_writer(args.slice(:delimiter)) do |io|
1126
+ klass.stream(io, **tabular_options(tabular, options), **reporting, &block)
401
1127
  end
402
1128
  end
403
1129
 
404
- def record_writer(delimiter: $/, **args, &block)
405
- return block.call(io_stream) if io_stream.is_a?(IOStreams::Record::Writer)
1130
+ # Yields the line writer that the rows or records of this stream are written to, in its format and by its schema,
1131
+ # with the supplied options: a line writer supplied to `IOStreams.stream`, which writes them in its own lines, see
1132
+ # #supplied_writer, otherwise one of this stream, see #line_writer.
1133
+ def rows_line_writer(args, &)
1134
+ return supplied_writer(IOStreams::Line::Writer, args, rows: true, &) if supplied_io.is_a?(IOStreams::Line::Writer)
406
1135
 
407
- line_writer(delimiter: delimiter) do |io|
408
- IOStreams::Record::Writer.stream(
409
- io,
410
- original_file_name: builder.file_name,
411
- format: builder.format,
412
- format_options: builder.format_options,
413
- **args,
414
- &block
415
- )
1136
+ line_writer(**args, &)
1137
+ end
1138
+
1139
+ # Yields this stream's IO, which is a writer of the supplied class that is already open, such as the one that
1140
+ # `writer(:line)` yields, handed to code that writes with `IOStreams.stream(io).writer(:line)`, which then writes into
1141
+ # it. It writes with its own settings, so it ignores the options of the call, the streams, options and encoding of
1142
+ # this stream, and its format and schema, other than those of the rows or records that are written to a line writer
1143
+ # (rows: true), which apply, see #rows_line_writer. Each is deprecated, since v3.0 raises ArgumentError for it, see
1144
+ # #warn_supplied_writer_options and #warn_supplied_writer_settings.
1145
+ def supplied_writer(klass, args, rows: false)
1146
+ warn_supplied_writer_options(klass, args)
1147
+ warn_supplied_writer_settings(klass, rows)
1148
+ yield(supplied_io)
1149
+ end
1150
+
1151
+ # Reports a deprecation warning for the options of a call that the supplied writer ignores, see #supplied_writer:
1152
+ # each of them, even when it is nil, other than a nil `delimiter:`, the only option of a writer of v3.0, which
1153
+ # accepts it, and raises ArgumentError for any other.
1154
+ def warn_supplied_writer_options(klass, args)
1155
+ names = args.reject { |name, value| name == :delimiter && value.nil? }.keys
1156
+ return if names.empty?
1157
+
1158
+ words = Deprecation::Words.new(names)
1159
+ warn_supplied_writer(klass, words.keywords, words)
1160
+ end
1161
+
1162
+ # Reports a deprecation warning for the settings of this stream that the supplied writer ignores, see
1163
+ # #supplied_writer, as the builder answers them, see `IOStreams::Builder#writer_settings`: the streams of its
1164
+ # pipeline, and, unless the rows or records of this stream are written to it, which apply them, its format and
1165
+ # schema.
1166
+ def warn_supplied_writer_settings(klass, rows)
1167
+ settings = builder.writer_settings
1168
+ streams = settings[:streams]
1169
+ unless streams.empty?
1170
+ words = Deprecation::Words.new(streams)
1171
+ warn_supplied_writer(klass, "the #{words.one? ? 'stream' : 'streams'} #{streams.map(&:inspect).join(', ')} " \
1172
+ "that #stream, #option, #encoding or #file_name set", words)
416
1173
  end
1174
+ return if rows || settings[:tabular].empty?
1175
+
1176
+ names = settings[:tabular].map { |name| "##{name}" }
1177
+ warn_supplied_writer(klass, names.join(" and "), Deprecation::Words.new(names))
1178
+ end
1179
+
1180
+ # Reports that what the subject names, which the words are about, see `IOStreams::Deprecation::Words`, is deprecated
1181
+ # for the supplied writer, which is already open, and ignores it.
1182
+ def warn_supplied_writer(klass, subject, words)
1183
+ Deprecation.warn("#{subject} #{words.are} deprecated for the #{klass.name} supplied to IOStreams.stream, which " \
1184
+ "is already open and writes with its own settings, so #{words.they_are} ignored: set " \
1185
+ "#{words.them} where that writer is opened instead, or remove #{words.them}")
417
1186
  end
418
1187
  end
419
1188
  end