iostreams 0.20.3 → 1.0.0.beta
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/io_streams/bzip2/reader.rb +9 -21
- data/lib/io_streams/bzip2/writer.rb +9 -21
- data/lib/io_streams/deprecated.rb +217 -0
- data/lib/io_streams/encode/reader.rb +12 -16
- data/lib/io_streams/encode/writer.rb +9 -13
- data/lib/io_streams/errors.rb +6 -6
- data/lib/io_streams/gzip/reader.rb +7 -14
- data/lib/io_streams/gzip/writer.rb +7 -15
- data/lib/io_streams/io_streams.rb +182 -524
- data/lib/io_streams/line/reader.rb +9 -9
- data/lib/io_streams/line/writer.rb +10 -11
- data/lib/io_streams/path.rb +190 -0
- data/lib/io_streams/paths/file.rb +176 -0
- data/lib/io_streams/paths/http.rb +92 -0
- data/lib/io_streams/paths/matcher.rb +61 -0
- data/lib/io_streams/paths/s3.rb +269 -0
- data/lib/io_streams/paths/sftp.rb +99 -0
- data/lib/io_streams/pgp.rb +47 -19
- data/lib/io_streams/pgp/reader.rb +20 -28
- data/lib/io_streams/pgp/writer.rb +24 -46
- data/lib/io_streams/reader.rb +28 -0
- data/lib/io_streams/record/reader.rb +20 -16
- data/lib/io_streams/record/writer.rb +28 -28
- data/lib/io_streams/row/reader.rb +22 -26
- data/lib/io_streams/row/writer.rb +29 -28
- data/lib/io_streams/stream.rb +400 -0
- data/lib/io_streams/streams.rb +125 -0
- data/lib/io_streams/symmetric_encryption/reader.rb +5 -13
- data/lib/io_streams/symmetric_encryption/writer.rb +16 -15
- data/lib/io_streams/tabular/header.rb +9 -3
- data/lib/io_streams/tabular/parser/array.rb +8 -3
- data/lib/io_streams/tabular/parser/csv.rb +6 -2
- data/lib/io_streams/tabular/parser/hash.rb +4 -1
- data/lib/io_streams/tabular/parser/json.rb +3 -1
- data/lib/io_streams/tabular/parser/psv.rb +3 -1
- data/lib/io_streams/tabular/utility/csv_row.rb +9 -8
- data/lib/io_streams/utils.rb +22 -0
- data/lib/io_streams/version.rb +1 -1
- data/lib/io_streams/writer.rb +28 -0
- data/lib/io_streams/xlsx/reader.rb +7 -19
- data/lib/io_streams/zip/reader.rb +7 -26
- data/lib/io_streams/zip/writer.rb +21 -38
- data/lib/iostreams.rb +15 -15
- data/test/bzip2_reader_test.rb +3 -3
- data/test/bzip2_writer_test.rb +3 -3
- data/test/deprecated_test.rb +123 -0
- data/test/encode_reader_test.rb +3 -3
- data/test/encode_writer_test.rb +6 -6
- data/test/gzip_reader_test.rb +2 -2
- data/test/gzip_writer_test.rb +3 -3
- data/test/io_streams_test.rb +43 -136
- data/test/line_reader_test.rb +20 -20
- data/test/line_writer_test.rb +3 -3
- data/test/path_test.rb +30 -28
- data/test/paths/file_test.rb +206 -0
- data/test/paths/http_test.rb +34 -0
- data/test/paths/matcher_test.rb +111 -0
- data/test/paths/s3_test.rb +207 -0
- data/test/pgp_reader_test.rb +8 -8
- data/test/pgp_writer_test.rb +13 -13
- data/test/record_reader_test.rb +5 -5
- data/test/record_writer_test.rb +4 -4
- data/test/row_reader_test.rb +5 -5
- data/test/row_writer_test.rb +6 -6
- data/test/stream_test.rb +116 -0
- data/test/streams_test.rb +255 -0
- data/test/utils_test.rb +20 -0
- data/test/xlsx_reader_test.rb +3 -3
- data/test/zip_reader_test.rb +12 -12
- data/test/zip_writer_test.rb +5 -5
- metadata +33 -45
- data/lib/io_streams/base_path.rb +0 -72
- data/lib/io_streams/file/path.rb +0 -58
- data/lib/io_streams/file/reader.rb +0 -12
- data/lib/io_streams/file/writer.rb +0 -22
- data/lib/io_streams/http/reader.rb +0 -71
- data/lib/io_streams/s3.rb +0 -26
- data/lib/io_streams/s3/path.rb +0 -40
- data/lib/io_streams/s3/reader.rb +0 -28
- data/lib/io_streams/s3/writer.rb +0 -85
- data/lib/io_streams/sftp/reader.rb +0 -67
- data/lib/io_streams/sftp/writer.rb +0 -68
- data/test/base_path_test.rb +0 -35
- data/test/file_path_test.rb +0 -97
- data/test/file_reader_test.rb +0 -33
- data/test/file_writer_test.rb +0 -50
- data/test/http_reader_test.rb +0 -38
- data/test/s3_reader_test.rb +0 -41
- data/test/s3_writer_test.rb +0 -41
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 158fef0e14167719cd27b7b1cb0445bfb08403a3afc4490fad74a0e0fc9ab67b
|
4
|
+
data.tar.gz: b1f3db75a4773591507706befa66122204ffda1642136c79427671a4bf240605
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 556a898a859c9447822cb0d3aeea58ce6d839536d43fe3ef46fdf94fb1794d50d596da713fd6eef4203ef84ed94790ad8d69581d54c6a167bc9cea57e4d6b976
|
7
|
+
data.tar.gz: 7c38374f8ec37252a3b5428472235cf44a3a6379b5190776d766f447f339f16f3f85b69713c77592601466f82eac998c227de8136cecc64f82081e00f540314b
|
@@ -1,28 +1,16 @@
|
|
1
1
|
module IOStreams
|
2
2
|
module Bzip2
|
3
|
-
class Reader
|
4
|
-
# Read from a Bzip2
|
5
|
-
def self.
|
6
|
-
|
7
|
-
require 'rbzip2' unless defined?(RBzip2)
|
8
|
-
rescue LoadError => e
|
9
|
-
raise(LoadError, "Please install the 'rbzip2' gem for Bzip2 streaming support. #{e.message}")
|
10
|
-
end
|
3
|
+
class Reader < IOStreams::Reader
|
4
|
+
# Read from a Bzip2 stream, decompressing the contents as it is read
|
5
|
+
def self.stream(input_stream, **_args)
|
6
|
+
Utils.load_dependency('rbzip2', 'Bzip2') unless defined?(RBzip2)
|
11
7
|
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
io.close if io && (io.respond_to?(:closed?) && !io.closed?)
|
18
|
-
end
|
19
|
-
else
|
20
|
-
::File.open(file_name_or_io, 'rb') do |file|
|
21
|
-
io = RBzip2.default_adapter::Decompressor.new(file)
|
22
|
-
block.call(io)
|
23
|
-
end
|
8
|
+
begin
|
9
|
+
io = RBzip2.default_adapter::Decompressor.new(input_stream)
|
10
|
+
yield io
|
11
|
+
ensure
|
12
|
+
io&.close
|
24
13
|
end
|
25
|
-
|
26
14
|
end
|
27
15
|
end
|
28
16
|
end
|
@@ -1,27 +1,15 @@
|
|
1
1
|
module IOStreams
|
2
2
|
module Bzip2
|
3
|
-
class Writer
|
4
|
-
# Write to a
|
5
|
-
def self.
|
6
|
-
|
7
|
-
require 'rbzip2' unless defined?(RBzip2)
|
8
|
-
rescue LoadError => e
|
9
|
-
raise(LoadError, "Please install the 'rbzip2' gem for Bzip2 streaming support. #{e.message}")
|
10
|
-
end
|
3
|
+
class Writer < IOStreams::Writer
|
4
|
+
# Write to a stream, compressing with Bzip2
|
5
|
+
def self.stream(input_stream, original_file_name: nil, **_args)
|
6
|
+
Utils.load_dependency('rbzip2', 'Bzip2') unless defined?(RBzip2)
|
11
7
|
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
io.close
|
18
|
-
end
|
19
|
-
else
|
20
|
-
IOStreams::File::Writer.open(file_name_or_io) do |file|
|
21
|
-
io = RBzip2.default_adapter::Compressor.new(file)
|
22
|
-
block.call(io)
|
23
|
-
io.close
|
24
|
-
end
|
8
|
+
begin
|
9
|
+
io = RBzip2.default_adapter::Compressor.new(input_stream)
|
10
|
+
yield io
|
11
|
+
ensure
|
12
|
+
io&.close
|
25
13
|
end
|
26
14
|
end
|
27
15
|
end
|
@@ -0,0 +1,217 @@
|
|
1
|
+
module IOStreams
|
2
|
+
UTF8_ENCODING = Encoding.find('UTF-8').freeze
|
3
|
+
BINARY_ENCODING = Encoding.find('BINARY').freeze
|
4
|
+
|
5
|
+
# Deprecated IOStreams from v0.x. Do not use, will be removed soon.
|
6
|
+
module Deprecated
|
7
|
+
def self.included(base)
|
8
|
+
base.extend ClassMethods
|
9
|
+
end
|
10
|
+
|
11
|
+
module ClassMethods
|
12
|
+
|
13
|
+
# DEPRECATED. Use `#path` or `#io`
|
14
|
+
# Examples:
|
15
|
+
# IOStreams.path("data.zip").reader { |f| f.read(100) }
|
16
|
+
#
|
17
|
+
# IOStreams.path(file_name).option(:encode, encoding: "BINARY").reader { |f| f.read(100) }
|
18
|
+
#
|
19
|
+
# io_stream = StringIO.new("Hello World")
|
20
|
+
# IOStreams.stream(io_stream).reader { |f| f.read(100) }
|
21
|
+
def reader(file_name_or_io, streams: nil, file_name: nil, encoding: nil, encode_cleaner: nil, encode_replace: nil, &block)
|
22
|
+
path = build_path(file_name_or_io, streams: streams, file_name: file_name, encoding: encoding, encode_cleaner: encode_cleaner, encode_replace: encode_replace)
|
23
|
+
path.reader(&block)
|
24
|
+
end
|
25
|
+
|
26
|
+
# DEPRECATED
|
27
|
+
def each_line(file_name_or_io, encoding: nil, encode_cleaner: nil, encode_replace: nil, **args, &block)
|
28
|
+
path = build_path(file_name_or_io, encoding: encoding, encode_cleaner: encode_cleaner, encode_replace: encode_replace)
|
29
|
+
path.each_line(**args, &block)
|
30
|
+
end
|
31
|
+
|
32
|
+
# DEPRECATED
|
33
|
+
def each_row(file_name_or_io, encoding: nil, encode_cleaner: nil, encode_replace: nil, **args, &block)
|
34
|
+
path = build_path(file_name_or_io, encoding: encoding, encode_cleaner: encode_cleaner, encode_replace: encode_replace)
|
35
|
+
path.each_row(**args, &block)
|
36
|
+
end
|
37
|
+
|
38
|
+
# DEPRECATED
|
39
|
+
def each_record(file_name_or_io, encoding: nil, encode_cleaner: nil, encode_replace: nil, **args, &block)
|
40
|
+
path = build_path(file_name_or_io, encoding: encoding, encode_cleaner: encode_cleaner, encode_replace: encode_replace)
|
41
|
+
path.each_record(**args, &block)
|
42
|
+
end
|
43
|
+
|
44
|
+
# DEPRECATED. Use `#path` or `#io`
|
45
|
+
# Examples:
|
46
|
+
# IOStreams.path("data.zip").writer { |f| f.write("Hello World") }
|
47
|
+
#
|
48
|
+
# IOStreams.path(file_name).option(:encode, encoding: "BINARY").writer { |f| f.write("Hello World") }
|
49
|
+
#
|
50
|
+
# io_stream = StringIO.new("Hello World")
|
51
|
+
# IOStreams.stream(io_stream).writer { |f| f.write("Hello World") }
|
52
|
+
def writer(file_name_or_io, streams: nil, file_name: nil, encoding: nil, encode_cleaner: nil, encode_replace: nil, &block)
|
53
|
+
path = build_path(file_name_or_io, streams: streams, file_name: file_name, encoding: encoding, encode_cleaner: encode_cleaner, encode_replace: encode_replace)
|
54
|
+
path.writer(&block)
|
55
|
+
end
|
56
|
+
|
57
|
+
# DEPRECATED
|
58
|
+
def line_writer(file_name_or_io, streams: nil, file_name: nil, encoding: nil, encode_cleaner: nil, encode_replace: nil, **args, &block)
|
59
|
+
path = build_path(file_name_or_io, streams: streams, file_name: file_name, encoding: encoding, encode_cleaner: encode_cleaner, encode_replace: encode_replace)
|
60
|
+
path.line_writer(**args, &block)
|
61
|
+
end
|
62
|
+
|
63
|
+
# DEPRECATED
|
64
|
+
def row_writer(file_name_or_io, streams: nil, file_name: nil, encoding: nil, encode_cleaner: nil, encode_replace: nil, **args, &block)
|
65
|
+
path = build_path(file_name_or_io, streams: streams, file_name: file_name, encoding: encoding, encode_cleaner: encode_cleaner, encode_replace: encode_replace)
|
66
|
+
path.row_writer(**args, &block)
|
67
|
+
end
|
68
|
+
|
69
|
+
# DEPRECATED
|
70
|
+
def record_writer(file_name_or_io, streams: nil, file_name: nil, encoding: nil, encode_cleaner: nil, encode_replace: nil, **args, &block)
|
71
|
+
path = build_path(file_name_or_io, streams: streams, file_name: file_name, encoding: encoding, encode_cleaner: encode_cleaner, encode_replace: encode_replace)
|
72
|
+
path.record_writer(**args, &block)
|
73
|
+
end
|
74
|
+
|
75
|
+
# Copies the source file/stream to the target file/stream.
|
76
|
+
# Returns [Integer] the number of bytes copied
|
77
|
+
#
|
78
|
+
# Example: Copy between 2 files
|
79
|
+
# IOStreams.copy('a.csv', 'b.csv')
|
80
|
+
#
|
81
|
+
# Example: Read content from a Xlsx file and write it out in CSV form.
|
82
|
+
# IOStreams.copy('a.xlsx', 'b.csv')
|
83
|
+
#
|
84
|
+
# Example:
|
85
|
+
# # Read content from a JSON file and write it out in CSV form.
|
86
|
+
# #
|
87
|
+
# # The output header for the CSV file is extracted from the first row in the JSON file.
|
88
|
+
# # If the first JSON row does not contain all the column names then they will be ignored
|
89
|
+
# # for the rest of the file.
|
90
|
+
# IOStreams.copy('a.json', 'b.csv')
|
91
|
+
#
|
92
|
+
# Example:
|
93
|
+
# # Read a PSV file and write out a CSV file from it.
|
94
|
+
# IOStreams.copy('a.psv', 'b.csv')
|
95
|
+
#
|
96
|
+
# Example:
|
97
|
+
# # Copy between 2 files, encrypting the target file with Symmetric Encryption
|
98
|
+
# # Since the target file_name already includes `.enc` in the filename, it is automatically
|
99
|
+
# # encrypted.
|
100
|
+
# IOStreams.copy('a.csv', 'b.csv.enc')
|
101
|
+
#
|
102
|
+
# Example:
|
103
|
+
# # Copy between 2 files, encrypting the target file with Symmetric Encryption
|
104
|
+
# # Since the target file_name does not include `.enc` in the filename, to encrypt it
|
105
|
+
# # the encryption stream is added.
|
106
|
+
# IOStreams.copy('a.csv', 'b', target_options: [:enc])
|
107
|
+
#
|
108
|
+
# Example:
|
109
|
+
# # Copy between 2 files, encrypting the target file with Symmetric Encryption
|
110
|
+
# # Since the target file_name does not include `.enc` in the filename, to encrypt it
|
111
|
+
# # the encryption stream is added, along with the optional compression option.
|
112
|
+
# IOStreams.copy('a.csv', 'b', target_options: [enc: { compress: true }])
|
113
|
+
#
|
114
|
+
# Example:
|
115
|
+
# # Create a pgp encrypted file.
|
116
|
+
# # For PGP Encryption the recipients email address is required.
|
117
|
+
# IOStreams.copy('a.xlsx', 'b.csv.pgp', target_options: [:csv, pgp: { recipient_email: 'user@nospam.org' }])
|
118
|
+
#
|
119
|
+
# Example: Copy between 2 existing streams
|
120
|
+
# IOStreams.reader('a.csv') do |source_stream|
|
121
|
+
# IOStreams.writer('b.csv.enc') do |target_stream|
|
122
|
+
# IOStreams.copy(source_stream, target_stream)
|
123
|
+
# end
|
124
|
+
# end
|
125
|
+
#
|
126
|
+
# Example:
|
127
|
+
# # Copy between 2 csv files, reducing the number of columns present and encrypting the
|
128
|
+
# # target file with Symmetric Encryption
|
129
|
+
# output_headers = %w[name address]
|
130
|
+
# IOStreams.copy(
|
131
|
+
# 'a.csv',
|
132
|
+
# 'b.csv.enc',
|
133
|
+
# target_options: [csv:{headers: output_headers}, enc: {compress: true}]
|
134
|
+
# )
|
135
|
+
#
|
136
|
+
# Example:
|
137
|
+
# # Copy a locally encrypted file to AWS S3.
|
138
|
+
# # Decrypts the file, then compresses it with gzip as it is being streamed into S3.
|
139
|
+
# # Useful for when the entire bucket is encrypted on S3.
|
140
|
+
# IOStreams.copy('a.csv.enc', 's3://my_bucket/b.csv.gz')
|
141
|
+
def copy(source_file_name_or_io, target_file_name_or_io, buffer_size: nil, source_options: {}, target_options: {})
|
142
|
+
# TODO: prevent stream conversions when reader and writer streams are the same!
|
143
|
+
reader(source_file_name_or_io, **source_options) do |source_stream|
|
144
|
+
writer(target_file_name_or_io, **target_options) do |target_stream|
|
145
|
+
IO.copy_stream(source_stream, target_stream)
|
146
|
+
end
|
147
|
+
end
|
148
|
+
end
|
149
|
+
|
150
|
+
# DEPRECATED
|
151
|
+
def reader_stream?(file_name_or_io)
|
152
|
+
file_name_or_io.respond_to?(:read)
|
153
|
+
end
|
154
|
+
|
155
|
+
# DEPRECATED
|
156
|
+
def writer_stream?(file_name_or_io)
|
157
|
+
file_name_or_io.respond_to?(:write)
|
158
|
+
end
|
159
|
+
|
160
|
+
# DEPRECATED. Use Path#compressed?
|
161
|
+
def compressed?(file_name)
|
162
|
+
Path.new(file_name).compressed?
|
163
|
+
end
|
164
|
+
|
165
|
+
# DEPRECATED. Use Path#encrypted?
|
166
|
+
def encrypted?(file_name)
|
167
|
+
Path.new(file_name).encrypted?
|
168
|
+
end
|
169
|
+
|
170
|
+
# DEPRECATED
|
171
|
+
def line_reader(file_name_or_io, streams: nil, file_name: nil, encoding: nil, encode_cleaner: nil, encode_replace: nil, **args, &block)
|
172
|
+
path = build_path(file_name_or_io, streams: streams, file_name: file_name, encoding: encoding, encode_cleaner: encode_cleaner, encode_replace: encode_replace)
|
173
|
+
path.line_reader(**args, &block)
|
174
|
+
end
|
175
|
+
|
176
|
+
# DEPRECATED
|
177
|
+
def row_reader(file_name_or_io, streams: nil, file_name: nil, encoding: nil, encode_cleaner: nil, encode_replace: nil, **args, &block)
|
178
|
+
path = build_path(file_name_or_io, streams: streams, file_name: file_name, encoding: encoding, encode_cleaner: encode_cleaner, encode_replace: encode_replace)
|
179
|
+
path.line_reader(**args, &block)
|
180
|
+
end
|
181
|
+
|
182
|
+
# DEPRECATED
|
183
|
+
def record_reader(file_name_or_io, streams: nil, file_name: nil, encoding: nil, encode_cleaner: nil, encode_replace: nil, **args, &block)
|
184
|
+
path = build_path(file_name_or_io, streams: streams, file_name: file_name, encoding: encoding, encode_cleaner: encode_cleaner, encode_replace: encode_replace)
|
185
|
+
path.record_reader(**args, &block)
|
186
|
+
end
|
187
|
+
|
188
|
+
private
|
189
|
+
|
190
|
+
def build_path(file_name_or_io, streams: nil, file_name: nil, encoding: nil, encode_cleaner: nil, encode_replace: nil)
|
191
|
+
path = new(file_name_or_io)
|
192
|
+
path.file_name(file_name) if file_name
|
193
|
+
|
194
|
+
apply_old_style_streams(path, streams) if streams
|
195
|
+
|
196
|
+
if encoding || encode_cleaner || encode_replace
|
197
|
+
path.option_or_stream(:encode, encoding: encoding, cleaner: encode_cleaner, replace: encode_replace)
|
198
|
+
end
|
199
|
+
|
200
|
+
path
|
201
|
+
end
|
202
|
+
|
203
|
+
# Applies old form streams to the path
|
204
|
+
def apply_old_style_streams(path, streams)
|
205
|
+
if streams.is_a?(Symbol)
|
206
|
+
path.stream(streams)
|
207
|
+
elsif streams.is_a?(Array)
|
208
|
+
streams.each { |stream| apply_old_style_streams(path, stream) }
|
209
|
+
elsif streams.is_a?(Hash)
|
210
|
+
streams.each_pair { |stream, options| path.stream(stream, options) }
|
211
|
+
else
|
212
|
+
raise ArgumentError, "Invalid old style stream supplied: #{params.inspect}"
|
213
|
+
end
|
214
|
+
end
|
215
|
+
end
|
216
|
+
end
|
217
|
+
end
|
@@ -1,22 +1,18 @@
|
|
1
1
|
module IOStreams
|
2
2
|
module Encode
|
3
|
-
class Reader
|
3
|
+
class Reader < IOStreams::Reader
|
4
4
|
attr_reader :encoding, :cleaner
|
5
5
|
|
6
6
|
NOT_PRINTABLE = Regexp.compile(/[^[:print:]|\r|\n]/).freeze
|
7
7
|
# Builtin strip options to apply after encoding the read data.
|
8
8
|
CLEANSE_RULES = {
|
9
9
|
# Strips all non printable characters
|
10
|
-
printable: ->
|
11
|
-
}
|
10
|
+
printable: ->(data) { data.gsub!(NOT_PRINTABLE, '') || data }
|
11
|
+
}.freeze
|
12
12
|
|
13
13
|
# Read a line at a time from a file or stream
|
14
|
-
def self.
|
15
|
-
|
16
|
-
IOStreams::File::Reader.open(file_name_or_io) { |io| yield new(io, **args) }
|
17
|
-
else
|
18
|
-
yield new(file_name_or_io, **args)
|
19
|
-
end
|
14
|
+
def self.stream(input_stream, original_file_name: nil, **args)
|
15
|
+
yield new(input_stream, **args)
|
20
16
|
end
|
21
17
|
|
22
18
|
# Apply encoding conversion when reading a stream.
|
@@ -33,26 +29,26 @@ module IOStreams
|
|
33
29
|
# Etc.
|
34
30
|
# Default: 'UTF-8'
|
35
31
|
#
|
36
|
-
#
|
32
|
+
# replace: [String]
|
37
33
|
# The character to replace with when a character cannot be converted to the target encoding.
|
38
34
|
# nil: Don't replace any invalid characters. Encoding::UndefinedConversionError is raised.
|
39
35
|
# Default: nil
|
40
36
|
#
|
41
|
-
#
|
37
|
+
# cleaner: [nil|symbol|Proc]
|
42
38
|
# Cleanse data read from the input stream.
|
43
39
|
# nil: No cleansing
|
44
40
|
# :printable Cleanse all non-printable characters except \r and \n
|
45
41
|
# Proc/lambda Proc to call after every read to cleanse the data
|
46
42
|
# Default: nil
|
47
|
-
def initialize(input_stream, encoding: 'UTF-8',
|
48
|
-
|
49
|
-
@cleaner = self.class.extract_cleaner(encode_cleaner)
|
43
|
+
def initialize(input_stream, encoding: 'UTF-8', cleaner: nil, replace: nil)
|
44
|
+
super(input_stream)
|
50
45
|
|
46
|
+
@cleaner = self.class.extract_cleaner(cleaner)
|
51
47
|
@encoding = encoding.nil? || encoding.is_a?(Encoding) ? encoding : Encoding.find(encoding)
|
52
|
-
@encoding_options =
|
48
|
+
@encoding_options = replace.nil? ? {} : {invalid: :replace, undef: :replace, replace: replace}
|
53
49
|
|
54
50
|
# More efficient read buffering only supported when the input stream `#read` method supports it.
|
55
|
-
if
|
51
|
+
if replace.nil? && !@input_stream.method(:read).arity.between?(0, 1)
|
56
52
|
@read_cache_buffer = ''.encode(@encoding)
|
57
53
|
else
|
58
54
|
@read_cache_buffer = nil
|
@@ -1,15 +1,11 @@
|
|
1
1
|
module IOStreams
|
2
2
|
module Encode
|
3
|
-
class Writer
|
3
|
+
class Writer < IOStreams::Writer
|
4
4
|
attr_reader :encoding, :cleaner
|
5
5
|
|
6
6
|
# Write a line at a time to a file or stream
|
7
|
-
def self.
|
8
|
-
|
9
|
-
IOStreams::File::Writer.open(file_name_or_io) { |io| yield new(io, **args) }
|
10
|
-
else
|
11
|
-
yield new(file_name_or_io, **args)
|
12
|
-
end
|
7
|
+
def self.stream(input_stream, original_file_name: nil, **args)
|
8
|
+
yield new(input_stream, **args)
|
13
9
|
end
|
14
10
|
|
15
11
|
# A delimited stream writer that will write to the supplied output stream
|
@@ -27,23 +23,23 @@ module IOStreams
|
|
27
23
|
# Etc.
|
28
24
|
# Default: 'UTF-8'
|
29
25
|
#
|
30
|
-
#
|
26
|
+
# replace: [String]
|
31
27
|
# The character to replace with when a character cannot be converted to the target encoding.
|
32
28
|
# nil: Don't replace any invalid characters. Encoding::UndefinedConversionError is raised.
|
33
29
|
# Default: nil
|
34
30
|
#
|
35
|
-
#
|
31
|
+
# cleaner: [nil|symbol|Proc]
|
36
32
|
# Cleanse data read from the input stream.
|
37
33
|
# nil: No cleansing
|
38
34
|
# :printable Cleanse all non-printable characters except \r and \n
|
39
35
|
# Proc/lambda Proc to call after every read to cleanse the data
|
40
36
|
# Default: nil
|
41
|
-
def initialize(output_stream, encoding: 'UTF-8',
|
42
|
-
|
43
|
-
@cleaner = ::IOStreams::Encode::Reader.send(:extract_cleaner, encode_cleaner)
|
37
|
+
def initialize(output_stream, encoding: 'UTF-8', cleaner: nil, replace: nil)
|
38
|
+
super(output_stream)
|
44
39
|
|
40
|
+
@cleaner = ::IOStreams::Encode::Reader.send(:extract_cleaner, cleaner)
|
45
41
|
@encoding = encoding.nil? || encoding.is_a?(Encoding) ? encoding : Encoding.find(encoding)
|
46
|
-
@encoding_options =
|
42
|
+
@encoding_options = replace.nil? ? {} : {invalid: :replace, undef: :replace, replace: replace}
|
47
43
|
end
|
48
44
|
|
49
45
|
# Write a line to the output stream
|
data/lib/io_streams/errors.rb
CHANGED
@@ -1,22 +1,22 @@
|
|
1
1
|
module IOStreams
|
2
2
|
module Errors
|
3
|
-
class Error < StandardError
|
3
|
+
class Error < StandardError
|
4
4
|
end
|
5
5
|
|
6
|
-
class InvalidHeader < Error
|
6
|
+
class InvalidHeader < Error
|
7
7
|
end
|
8
8
|
|
9
|
-
class MissingHeader < Error
|
9
|
+
class MissingHeader < Error
|
10
10
|
end
|
11
11
|
|
12
|
-
class TypeMismatch < Error
|
12
|
+
class TypeMismatch < Error
|
13
13
|
end
|
14
14
|
|
15
|
-
class CommunicationsFailure < Error
|
15
|
+
class CommunicationsFailure < Error
|
16
16
|
end
|
17
17
|
|
18
18
|
# When the specified delimiter is not found in the supplied stream / file
|
19
|
-
class DelimiterNotFound < Error
|
19
|
+
class DelimiterNotFound < Error
|
20
20
|
end
|
21
21
|
end
|
22
22
|
end
|
@@ -1,20 +1,13 @@
|
|
1
1
|
module IOStreams
|
2
2
|
module Gzip
|
3
|
-
class Reader
|
4
|
-
# Read from a gzip
|
5
|
-
def self.
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
io = ::Zlib::GzipReader.new(file_name_or_io)
|
11
|
-
block.call(io)
|
12
|
-
ensure
|
13
|
-
io.close if io && (io.respond_to?(:closed?) && !io.closed?)
|
14
|
-
end
|
15
|
-
end
|
3
|
+
class Reader < IOStreams::Reader
|
4
|
+
# Read from a gzip stream, decompressing the contents as it is read
|
5
|
+
def self.stream(input_stream, original_file_name: nil, &block)
|
6
|
+
io = ::Zlib::GzipReader.new(input_stream)
|
7
|
+
block.call(io)
|
8
|
+
ensure
|
9
|
+
io&.close
|
16
10
|
end
|
17
|
-
|
18
11
|
end
|
19
12
|
end
|
20
13
|
end
|