iostreams 1.0.0.beta7 → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +138 -78
- data/lib/io_streams/{streams.rb → builder.rb} +4 -3
- data/lib/io_streams/deprecated.rb +9 -9
- data/lib/io_streams/io_streams.rb +11 -11
- data/lib/io_streams/line/writer.rb +2 -2
- data/lib/io_streams/path.rb +7 -7
- data/lib/io_streams/paths/file.rb +13 -4
- data/lib/io_streams/paths/http.rb +4 -2
- data/lib/io_streams/paths/s3.rb +4 -4
- data/lib/io_streams/paths/sftp.rb +37 -36
- data/lib/io_streams/pgp/reader.rb +3 -1
- data/lib/io_streams/record/writer.rb +1 -1
- data/lib/io_streams/row/writer.rb +1 -1
- data/lib/io_streams/stream.rb +134 -109
- data/lib/io_streams/version.rb +1 -1
- data/lib/iostreams.rb +1 -1
- data/test/{streams_test.rb → builder_test.rb} +23 -11
- data/test/encode_reader_test.rb +1 -0
- data/test/paths/s3_test.rb +1 -1
- data/test/stream_test.rb +26 -26
- metadata +8 -8
@@ -37,15 +37,15 @@ module IOStreams
|
|
37
37
|
# # => "files/file.xls"
|
38
38
|
#
|
39
39
|
# For Files
|
40
|
-
# IOStreams.path('blah.zip').option(:encode, encoding: 'BINARY').
|
41
|
-
# IOStreams.path('blah.zip').option(:encode, encoding: 'UTF-8').
|
42
|
-
# IOStreams.path('blah.zip').option(:encode, encoding: 'UTF-8').
|
43
|
-
# IOStreams.path('blah.zip').option(:encode, encoding: 'UTF-8').
|
40
|
+
# IOStreams.path('blah.zip').option(:encode, encoding: 'BINARY').each(:line) { |line| puts line }
|
41
|
+
# IOStreams.path('blah.zip').option(:encode, encoding: 'UTF-8').each(:line).first
|
42
|
+
# IOStreams.path('blah.zip').option(:encode, encoding: 'UTF-8').each(:hash).last
|
43
|
+
# IOStreams.path('blah.zip').option(:encode, encoding: 'UTF-8').each(:hash).size
|
44
44
|
# IOStreams.path('blah.zip').option(:encode, encoding: 'UTF-8').reader.size
|
45
|
-
# IOStreams.path('blah.csv.zip').
|
46
|
-
# IOStreams.path('blah.zip').option(:pgp, passphrase: 'receiver_passphrase').
|
47
|
-
# IOStreams.path('blah.zip').stream(:zip).stream(:pgp, passphrase: 'receiver_passphrase').
|
48
|
-
# IOStreams.path('blah.zip').stream(:zip).stream(:encode, encoding: 'BINARY').
|
45
|
+
# IOStreams.path('blah.csv.zip').each(:line) { |line| puts line }
|
46
|
+
# IOStreams.path('blah.zip').option(:pgp, passphrase: 'receiver_passphrase').read
|
47
|
+
# IOStreams.path('blah.zip').stream(:zip).stream(:pgp, passphrase: 'receiver_passphrase').read
|
48
|
+
# IOStreams.path('blah.zip').stream(:zip).stream(:encode, encoding: 'BINARY').read
|
49
49
|
#
|
50
50
|
def self.path(*elements, **args)
|
51
51
|
return elements.first if (elements.size == 1) && args.empty? && elements.first.is_a?(IOStreams::Path)
|
@@ -59,9 +59,9 @@ module IOStreams
|
|
59
59
|
|
60
60
|
# For an existing IO Stream
|
61
61
|
# IOStreams.stream(io).file_name('blah.zip').encoding('BINARY').reader(&:read)
|
62
|
-
# IOStreams.stream(io).file_name('blah.zip').encoding('BINARY').
|
63
|
-
# IOStreams.stream(io).file_name('blah.csv.zip').
|
64
|
-
# IOStreams.stream(io).stream(:zip).stream(:pgp, passphrase: 'receiver_passphrase').
|
62
|
+
# IOStreams.stream(io).file_name('blah.zip').encoding('BINARY').each(:line){ ... }
|
63
|
+
# IOStreams.stream(io).file_name('blah.csv.zip').each(:line) { ... }
|
64
|
+
# IOStreams.stream(io).stream(:zip).stream(:pgp, passphrase: 'receiver_passphrase').read
|
65
65
|
def self.stream(io_stream)
|
66
66
|
return io_stream if io_stream.is_a?(Stream)
|
67
67
|
|
@@ -32,7 +32,7 @@ module IOStreams
|
|
32
32
|
# Write a line to the output stream
|
33
33
|
#
|
34
34
|
# Example:
|
35
|
-
# IOStreams.
|
35
|
+
# IOStreams.path('a.txt').writer(:line) do |stream|
|
36
36
|
# stream << 'first line' << 'second line'
|
37
37
|
# end
|
38
38
|
def <<(data)
|
@@ -44,7 +44,7 @@ module IOStreams
|
|
44
44
|
# Returns [Integer] the number of bytes written.
|
45
45
|
#
|
46
46
|
# Example:
|
47
|
-
# IOStreams.
|
47
|
+
# IOStreams.path('a.txt').writer(:line) do |stream|
|
48
48
|
# count = stream.write('first line')
|
49
49
|
# puts "Wrote #{count} bytes to the output file, including the delimiter"
|
50
50
|
# end
|
data/lib/io_streams/path.rb
CHANGED
@@ -8,7 +8,7 @@ module IOStreams
|
|
8
8
|
|
9
9
|
@path = path.frozen? ? path : path.dup.freeze
|
10
10
|
@io_stream = nil
|
11
|
-
@
|
11
|
+
@builder = nil
|
12
12
|
end
|
13
13
|
|
14
14
|
# If elements already contains the current path then it is used as is without
|
@@ -20,7 +20,7 @@ module IOStreams
|
|
20
20
|
relative = ::File.join(*elements)
|
21
21
|
|
22
22
|
new_path = dup
|
23
|
-
new_path.
|
23
|
+
new_path.builder = nil
|
24
24
|
new_path.path = relative.start_with?(path) ? relative : ::File.join(path, relative)
|
25
25
|
new_path
|
26
26
|
end
|
@@ -117,7 +117,7 @@ module IOStreams
|
|
117
117
|
# IOStreams.path(".profile").directory #=> "."
|
118
118
|
def directory
|
119
119
|
new_path = dup
|
120
|
-
new_path.
|
120
|
+
new_path.builder = nil
|
121
121
|
new_path.path = ::File.dirname(path)
|
122
122
|
new_path
|
123
123
|
end
|
@@ -188,15 +188,15 @@ module IOStreams
|
|
188
188
|
|
189
189
|
def inspect
|
190
190
|
str = "#<#{self.class.name}:#{path}"
|
191
|
-
str << " @
|
192
|
-
str << " @options=#{
|
191
|
+
str << " @builder=#{builder.streams.inspect}" if builder.streams
|
192
|
+
str << " @options=#{builder.options.inspect}" if builder.options
|
193
193
|
str << " pipeline=#{pipeline.inspect}>"
|
194
194
|
end
|
195
195
|
|
196
196
|
private
|
197
197
|
|
198
|
-
def
|
199
|
-
@
|
198
|
+
def builder
|
199
|
+
@builder ||= IOStreams::Builder.new(path)
|
200
200
|
end
|
201
201
|
end
|
202
202
|
end
|
@@ -3,6 +3,13 @@ require "fileutils"
|
|
3
3
|
module IOStreams
|
4
4
|
module Paths
|
5
5
|
class File < IOStreams::Path
|
6
|
+
attr_accessor :create_path
|
7
|
+
|
8
|
+
def initialize(file_name, create_path: true)
|
9
|
+
@create_path = create_path
|
10
|
+
super(file_name)
|
11
|
+
end
|
12
|
+
|
6
13
|
# Yields Paths within the current path.
|
7
14
|
#
|
8
15
|
# Examples:
|
@@ -146,9 +153,11 @@ module IOStreams
|
|
146
153
|
self.class.new(::File.realpath(path))
|
147
154
|
end
|
148
155
|
|
156
|
+
private
|
157
|
+
|
149
158
|
# Read from file
|
150
|
-
def
|
151
|
-
::File.open(path, "rb") { |io|
|
159
|
+
def stream_reader(&block)
|
160
|
+
::File.open(path, "rb") { |io| builder.reader(io, &block) }
|
152
161
|
end
|
153
162
|
|
154
163
|
# Write to file
|
@@ -156,10 +165,10 @@ module IOStreams
|
|
156
165
|
# Note:
|
157
166
|
# If an exception is raised whilst the file is being written to the file is removed to
|
158
167
|
# prevent incomplete / partial files from being created.
|
159
|
-
def
|
168
|
+
def stream_writer(&block)
|
160
169
|
mkpath if create_path
|
161
170
|
begin
|
162
|
-
::File.open(path, "wb") { |io|
|
171
|
+
::File.open(path, "wb") { |io| builder.writer(io, &block) }
|
163
172
|
rescue StandardError => e
|
164
173
|
::File.unlink(path) if ::File.exist?(path)
|
165
174
|
raise(e)
|
@@ -47,6 +47,8 @@ module IOStreams
|
|
47
47
|
url
|
48
48
|
end
|
49
49
|
|
50
|
+
private
|
51
|
+
|
50
52
|
# Read a file using an http get.
|
51
53
|
#
|
52
54
|
# For example:
|
@@ -57,7 +59,7 @@ module IOStreams
|
|
57
59
|
#
|
58
60
|
# Notes:
|
59
61
|
# * Since Net::HTTP download only supports a push stream, the data is streamed into a tempfile first.
|
60
|
-
def
|
62
|
+
def stream_reader(&block)
|
61
63
|
handle_redirects(url, http_redirect_count, &block)
|
62
64
|
end
|
63
65
|
|
@@ -91,7 +93,7 @@ module IOStreams
|
|
91
93
|
Utils.temp_file_name('iostreams_http') do |file_name|
|
92
94
|
::File.open(file_name, 'wb') { |io| response.read_body { |chunk| io.write(chunk) } }
|
93
95
|
# Return a read stream
|
94
|
-
result = ::File.open(file_name, "rb") { |io|
|
96
|
+
result = ::File.open(file_name, "rb") { |io| builder.reader(io, &block) }
|
95
97
|
end
|
96
98
|
end
|
97
99
|
end
|
data/lib/io_streams/paths/s3.rb
CHANGED
@@ -208,12 +208,12 @@ module IOStreams
|
|
208
208
|
# TODO: delete_all
|
209
209
|
|
210
210
|
# Read from AWS S3 file.
|
211
|
-
def
|
211
|
+
def stream_reader(&block)
|
212
212
|
# Since S3 download only supports a push stream, write it to a tempfile first.
|
213
213
|
Utils.temp_file_name("iostreams_s3") do |file_name|
|
214
214
|
read_file(file_name)
|
215
215
|
|
216
|
-
::File.open(file_name, "rb") { |io|
|
216
|
+
::File.open(file_name, "rb") { |io| builder.reader(io, &block) }
|
217
217
|
end
|
218
218
|
end
|
219
219
|
|
@@ -231,10 +231,10 @@ module IOStreams
|
|
231
231
|
# aborted and this error is raised. The raised error has a `#errors`
|
232
232
|
# method that returns the failures that caused the upload to be
|
233
233
|
# aborted.
|
234
|
-
def
|
234
|
+
def stream_writer(&block)
|
235
235
|
# Since S3 upload only supports a pull stream, write it to a tempfile first.
|
236
236
|
Utils.temp_file_name("iostreams_s3") do |file_name|
|
237
|
-
result = ::File.open(file_name, "wb") { |io|
|
237
|
+
result = ::File.open(file_name, "wb") { |io| builder.writer(io, &block) }
|
238
238
|
|
239
239
|
# Upload file only once all data has been written to it
|
240
240
|
write_file(file_name)
|
@@ -2,6 +2,26 @@ require 'open3'
|
|
2
2
|
|
3
3
|
module IOStreams
|
4
4
|
module Paths
|
5
|
+
# Read a file from a remote sftp server.
|
6
|
+
#
|
7
|
+
# Example:
|
8
|
+
# IOStreams.
|
9
|
+
# path("sftp://example.org/path/file.txt", username: "jbloggs", password: "secret", compression: false).
|
10
|
+
# reader do |input|
|
11
|
+
# puts input.read
|
12
|
+
# end
|
13
|
+
#
|
14
|
+
# Note:
|
15
|
+
# - raises Net::SFTP::StatusException when the file could not be read.
|
16
|
+
#
|
17
|
+
# Write to a file on a remote sftp server.
|
18
|
+
#
|
19
|
+
# Example:
|
20
|
+
# IOStreams.
|
21
|
+
# path("sftp://example.org/path/file.txt", username: "jbloggs", password: "secret", compression: false).
|
22
|
+
# writer do |output|
|
23
|
+
# output.write('Hello World')
|
24
|
+
# end
|
5
25
|
class SFTP < IOStreams::Path
|
6
26
|
include SemanticLogger::Loggable if defined?(SemanticLogger)
|
7
27
|
|
@@ -54,7 +74,7 @@ module IOStreams
|
|
54
74
|
# IOStreams.path("sftp://test.com/path/file_name.csv", username: "jack", IdentityFile: "~/.ssh/private_key").reader do |io|
|
55
75
|
# puts io.read
|
56
76
|
# end
|
57
|
-
def initialize(url, username: nil, password: nil,
|
77
|
+
def initialize(url, username: nil, password: nil, ssh_options: {})
|
58
78
|
uri = Utils::URI.new(url)
|
59
79
|
raise(ArgumentError, "Invalid URL. Required Format: 'sftp://<host_name>/<file_name>'") unless uri.scheme == 'sftp'
|
60
80
|
|
@@ -87,40 +107,6 @@ module IOStreams
|
|
87
107
|
self
|
88
108
|
end
|
89
109
|
|
90
|
-
# Read a file from a remote sftp server.
|
91
|
-
#
|
92
|
-
# Example:
|
93
|
-
# IOStreams.
|
94
|
-
# path("sftp://example.org/path/file.txt", username: "jbloggs", password: "secret", compression: false).
|
95
|
-
# reader do |input|
|
96
|
-
# puts input.read
|
97
|
-
# end
|
98
|
-
#
|
99
|
-
# Note:
|
100
|
-
# - raises Net::SFTP::StatusException when the file could not be read.
|
101
|
-
def reader(&block)
|
102
|
-
IOStreams.temp_file("iostreams-sftp-reader") do |temp_file|
|
103
|
-
sftp_download(path, temp_file.to_s)
|
104
|
-
::File.open(temp_file.to_s, "rb") { |io| streams.reader(io, &block) }
|
105
|
-
end
|
106
|
-
end
|
107
|
-
|
108
|
-
# Write to a file on a remote sftp server.
|
109
|
-
#
|
110
|
-
# Example:
|
111
|
-
# IOStreams.
|
112
|
-
# path("sftp://example.org/path/file.txt", username: "jbloggs", password: "secret", compression: false).
|
113
|
-
# writer do |output|
|
114
|
-
# output.write('Hello World')
|
115
|
-
# end
|
116
|
-
def writer(&block)
|
117
|
-
IOStreams.temp_file("iostreams-sftp-writer") do |temp_file|
|
118
|
-
::File.open(temp_file.to_s, "wb") { |io| streams.writer(io, &block) }
|
119
|
-
sftp_upload(temp_file.to_s, path)
|
120
|
-
temp_file.size
|
121
|
-
end
|
122
|
-
end
|
123
|
-
|
124
110
|
# TODO: Add #copy_from shortcut to detect when a file is supplied that does not require conversion.
|
125
111
|
|
126
112
|
# Search for files on the remote sftp server that match the provided pattern.
|
@@ -147,7 +133,7 @@ module IOStreams
|
|
147
133
|
Net::SFTP.start(hostname, username, build_ssh_options) do |sftp|
|
148
134
|
sftp.dir.glob(".", pattern, flags) do |path|
|
149
135
|
next if !directories && !path.file?
|
150
|
-
new_path = self.class.new("sftp://#{hostname}/#{path.name}", username: username, password: password,
|
136
|
+
new_path = self.class.new("sftp://#{hostname}/#{path.name}", username: username, password: password, **ssh_options)
|
151
137
|
yield(new_path, path.attributes.attributes)
|
152
138
|
end
|
153
139
|
end
|
@@ -158,6 +144,21 @@ module IOStreams
|
|
158
144
|
|
159
145
|
attr_reader :password
|
160
146
|
|
147
|
+
def stream_reader(&block)
|
148
|
+
IOStreams.temp_file("iostreams-sftp-reader") do |temp_file|
|
149
|
+
sftp_download(path, temp_file.to_s)
|
150
|
+
::File.open(temp_file.to_s, "rb") { |io| builder.reader(io, &block) }
|
151
|
+
end
|
152
|
+
end
|
153
|
+
|
154
|
+
def stream_writer(&block)
|
155
|
+
IOStreams.temp_file("iostreams-sftp-writer") do |temp_file|
|
156
|
+
::File.open(temp_file.to_s, "wb") { |io| builder.writer(io, &block) }
|
157
|
+
sftp_upload(temp_file.to_s, path)
|
158
|
+
temp_file.size
|
159
|
+
end
|
160
|
+
end
|
161
|
+
|
161
162
|
# Use sftp and sshpass executables to download to a local file
|
162
163
|
def sftp_download(remote_file_name, local_file_name)
|
163
164
|
with_sftp_args do |args|
|
@@ -21,7 +21,9 @@ module IOStreams
|
|
21
21
|
#
|
22
22
|
# passphrase: [String]
|
23
23
|
# Pass phrase for private key to decrypt the file with
|
24
|
-
def self.file(file_name, passphrase:
|
24
|
+
def self.file(file_name, passphrase: nil)
|
25
|
+
# Cannot use `passphrase: self.default_passphrase` since it is considered private
|
26
|
+
passphrase ||= default_passphrase
|
25
27
|
raise(ArgumentError, 'Missing both passphrase and IOStreams::Pgp::Reader.default_passphrase') unless passphrase
|
26
28
|
|
27
29
|
loopback = IOStreams::Pgp.pgp_version.to_f >= 2.1 ? '--pinentry-mode loopback' : ''
|
@@ -1,7 +1,7 @@
|
|
1
1
|
module IOStreams
|
2
2
|
module Record
|
3
3
|
# Example, implied header from first record:
|
4
|
-
# IOStreams.path('file.csv').
|
4
|
+
# IOStreams.path('file.csv').writer(:hash) do |stream|
|
5
5
|
# stream << {name: 'Jack', address: 'Somewhere', zipcode: 12345}
|
6
6
|
# stream << {name: 'Joe', address: 'Lost', zipcode: 32443, age: 23}
|
7
7
|
# end
|
@@ -2,7 +2,7 @@ require 'csv'
|
|
2
2
|
module IOStreams
|
3
3
|
module Row
|
4
4
|
# Example:
|
5
|
-
# IOStreams.path("file.csv").
|
5
|
+
# IOStreams.path("file.csv").writer(:array) do |stream|
|
6
6
|
# stream << ['name', 'address', 'zipcode']
|
7
7
|
# stream << ['Jack', 'Somewhere', 12345]
|
8
8
|
# stream << ['Joe', 'Lost', 32443]
|
data/lib/io_streams/stream.rb
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
module IOStreams
|
2
2
|
class Stream
|
3
3
|
attr_reader :io_stream
|
4
|
-
attr_writer :
|
4
|
+
attr_writer :builder
|
5
5
|
|
6
6
|
def initialize(io_stream)
|
7
7
|
raise(ArgumentError, 'io_stream cannot be nil') if io_stream.nil?
|
8
8
|
raise(ArgumentError, "io_stream must not be a string: #{io_stream.inspect}") if io_stream.is_a?(String)
|
9
9
|
|
10
10
|
@io_stream = io_stream
|
11
|
-
@
|
11
|
+
@builder = nil
|
12
12
|
end
|
13
13
|
|
14
14
|
# Ignore the filename and use only the supplied streams.
|
@@ -19,7 +19,7 @@ module IOStreams
|
|
19
19
|
#
|
20
20
|
# IOStreams.path('tempfile2527').stream(:zip).stream(:pgp, passphrase: 'receiver_passphrase').reader(&:read)
|
21
21
|
def stream(stream, **options)
|
22
|
-
|
22
|
+
builder.stream(stream, **options)
|
23
23
|
self
|
24
24
|
end
|
25
25
|
|
@@ -34,32 +34,76 @@ module IOStreams
|
|
34
34
|
#
|
35
35
|
# IOStreams.path(output_file_name).option(:pgp, passphrase: 'receiver_passphrase').reader(&:read)
|
36
36
|
def option(stream, **options)
|
37
|
-
|
37
|
+
builder.option(stream, **options)
|
38
38
|
self
|
39
39
|
end
|
40
40
|
|
41
41
|
# Adds the options for the specified stream as an option,
|
42
42
|
# but if streams have already been added it is instead added as a stream.
|
43
43
|
def option_or_stream(stream, **options)
|
44
|
-
|
44
|
+
builder.option_or_stream(stream, **options)
|
45
45
|
self
|
46
46
|
end
|
47
47
|
|
48
48
|
# Return the options already set for either a stream or option.
|
49
49
|
def setting(stream)
|
50
|
-
|
50
|
+
builder.setting(stream)
|
51
51
|
self
|
52
52
|
end
|
53
53
|
|
54
54
|
# Returns [Hash<Symbol:Hash>] the pipeline of streams
|
55
55
|
# with their options that will be applied when the reader or writer is invoked.
|
56
56
|
def pipeline
|
57
|
-
|
57
|
+
builder.pipeline
|
58
|
+
end
|
59
|
+
|
60
|
+
# Iterate over a file / stream returning one line at a time.
|
61
|
+
#
|
62
|
+
# Example: Read a line at a time
|
63
|
+
# IOStreams.path("file.txt").each(:line) do |line|
|
64
|
+
# puts line
|
65
|
+
# end
|
66
|
+
#
|
67
|
+
# Example: Read a line at a time with custom options
|
68
|
+
# IOStreams.path("file.csv").each(:line, embedded_within: '"') do |line|
|
69
|
+
# puts line
|
70
|
+
# end
|
71
|
+
#
|
72
|
+
# Example: Read a row at a time
|
73
|
+
# IOStreams.path("file.csv").each(:array) do |array|
|
74
|
+
# p array
|
75
|
+
# end
|
76
|
+
#
|
77
|
+
# Example: Read a record at a time
|
78
|
+
# IOStreams.path("file.csv").each(:hash) do |hash|
|
79
|
+
# p hash
|
80
|
+
# end
|
81
|
+
#
|
82
|
+
# Notes:
|
83
|
+
# - Embedded lines (within double quotes) will be skipped if
|
84
|
+
# 1. The file name contains .csv
|
85
|
+
# 2. Or the embedded_within argument is set
|
86
|
+
def each(mode = :line, **args, &block)
|
87
|
+
raise(ArgumentError, "Invalid mode: #{mode.inspect}") if mode == :stream
|
88
|
+
|
89
|
+
# return enum_for __method__ unless block_given?
|
90
|
+
reader(mode, **args) { |stream| stream.each(&block) }
|
58
91
|
end
|
59
92
|
|
60
93
|
# Returns a Reader for reading a file / stream
|
61
|
-
def reader(&block)
|
62
|
-
|
94
|
+
def reader(mode = :stream, **args, &block)
|
95
|
+
case mode
|
96
|
+
when :stream
|
97
|
+
stream_reader(&block)
|
98
|
+
when :line
|
99
|
+
line_reader(**args, &block)
|
100
|
+
when :array
|
101
|
+
row_reader(**args, &block)
|
102
|
+
when :hash
|
103
|
+
record_reader(**args, &block)
|
104
|
+
else
|
105
|
+
raise(ArgumentError, "Invalid mode: #{mode.inspect}")
|
106
|
+
end
|
63
107
|
end
|
64
108
|
|
65
109
|
# Read an entire file into memory.
|
@@ -67,12 +111,37 @@ module IOStreams
|
|
67
111
|
# Notes:
|
68
112
|
# - Use with caution since large files can cause a denial of service since
|
69
113
|
# this method will load the entire file into memory.
|
70
|
-
# - Recommend using instead `#reader
|
71
|
-
# block into memory at a time.
|
114
|
+
# - Recommend using instead `#reader` to read a block into memory at a time.
|
72
115
|
def read(*args)
|
73
116
|
reader { |stream| stream.read(*args) }
|
74
117
|
end
|
75
118
|
|
119
|
+
# Returns a Writer for writing to a file / stream
|
120
|
+
def writer(mode = :stream, **args, &block)
|
121
|
+
case mode
|
122
|
+
when :stream
|
123
|
+
stream_writer(&block)
|
124
|
+
when :line
|
125
|
+
line_writer(**args, &block)
|
126
|
+
when :array
|
127
|
+
row_writer(**args, &block)
|
128
|
+
when :hash
|
129
|
+
record_writer(**args, &block)
|
130
|
+
else
|
131
|
+
raise(ArgumentError, "Invalid mode: #{mode.inspect}")
|
132
|
+
end
|
133
|
+
end
|
134
|
+
|
135
|
+
# Write entire string to file.
|
136
|
+
#
|
137
|
+
# Notes:
|
138
|
+
# - Use with caution since preparing large amounts of data in memory can cause a denial of service
|
139
|
+
# since all the data for the file needs to be resident in memory before writing.
|
140
|
+
# - Recommend using instead `#writer` to write a block of memory at a time.
|
141
|
+
def write(data)
|
142
|
+
writer { |stream| stream.write(data) }
|
143
|
+
end
|
144
|
+
|
76
145
|
# Copy from another stream, path, file_name or IO instance.
|
77
146
|
#
|
78
147
|
# Parameters:
|
@@ -113,109 +182,19 @@ module IOStreams
|
|
113
182
|
target.copy_from(self, convert: convert)
|
114
183
|
end
|
115
184
|
|
116
|
-
# Iterate over a file / stream returning one line at a time.
|
117
|
-
# Embedded lines (within double quotes) will be skipped if
|
118
|
-
# 1. The file name contains .csv
|
119
|
-
# 2. Or the embedded_within argument is set
|
120
|
-
#
|
121
|
-
# Example: Supply custom options
|
122
|
-
# IOStreams.each_line(file_name, embedded_within: '"') do |line|
|
123
|
-
# puts line
|
124
|
-
# end
|
125
|
-
#
|
126
|
-
def each_line(**args, &block)
|
127
|
-
# return enum_for __method__ unless block_given?
|
128
|
-
line_reader(**args) { |line_stream| line_stream.each(&block) }
|
129
|
-
end
|
130
|
-
|
131
|
-
# Iterate over a file / stream returning one line at a time.
|
132
|
-
# Embedded lines (within double quotes) will be skipped if
|
133
|
-
# 1. The file name contains .csv
|
134
|
-
# 2. Or the embedded_within argument is set
|
135
|
-
#
|
136
|
-
# Example: Supply custom options
|
137
|
-
# IOStreams.each_row(file_name, embedded_within: '"') do |line|
|
138
|
-
# puts line
|
139
|
-
# end
|
140
|
-
#
|
141
|
-
def each_row(**args, &block)
|
142
|
-
row_reader(**args) { |row_stream| row_stream.each(&block) }
|
143
|
-
end
|
144
|
-
|
145
|
-
# Returns [Hash] of every record in a file or stream with support for headers.
|
146
|
-
def each_record(**args, &block)
|
147
|
-
record_reader(**args) { |record_stream| record_stream.each(&block) }
|
148
|
-
end
|
149
|
-
|
150
|
-
# Iterate over a file / stream returning each record/line one at a time.
|
151
|
-
# It will apply the embedded_within argument if the file or input_stream contain .csv in its name.
|
152
|
-
def line_reader(embedded_within: nil, **args)
|
153
|
-
embedded_within = '"' if embedded_within.nil? && streams.file_name&.include?('.csv')
|
154
|
-
|
155
|
-
reader { |io| yield IOStreams::Line::Reader.new(io, embedded_within: embedded_within, **args) }
|
156
|
-
end
|
157
|
-
|
158
|
-
# Iterate over a file / stream returning each line as an array, one at a time.
|
159
|
-
def row_reader(delimiter: nil, embedded_within: nil, **args)
|
160
|
-
line_reader(delimiter: delimiter, embedded_within: embedded_within) do |io|
|
161
|
-
yield IOStreams::Row::Reader.new(io, **args)
|
162
|
-
end
|
163
|
-
end
|
164
|
-
|
165
|
-
# Iterate over a file / stream returning each line as a hash, one at a time.
|
166
|
-
def record_reader(delimiter: nil, embedded_within: nil, **args)
|
167
|
-
line_reader(delimiter: delimiter, embedded_within: embedded_within) do |io|
|
168
|
-
yield IOStreams::Record::Reader.new(io, **args)
|
169
|
-
end
|
170
|
-
end
|
171
|
-
|
172
|
-
# Returns a Writer for writing to a file / stream
|
173
|
-
def writer(&block)
|
174
|
-
streams.writer(io_stream, &block)
|
175
|
-
end
|
176
|
-
|
177
|
-
# Write entire string to file.
|
178
|
-
#
|
179
|
-
# Notes:
|
180
|
-
# - Use with caution since preparing large amounts of data in memory can cause a denial of service
|
181
|
-
# since all the data for the file needs to be resident in memory before writing.
|
182
|
-
# - Recommend using instead `#writer`, `#line_writer`, or `#row_writer` to write a
|
183
|
-
# block of memory at a time.
|
184
|
-
def write(data)
|
185
|
-
writer { |stream| stream.write(data) }
|
186
|
-
end
|
187
|
-
|
188
|
-
def line_writer(**args, &block)
|
189
|
-
return block.call(io_stream) if io_stream && io_stream.is_a?(IOStreams::Line::Writer)
|
190
|
-
|
191
|
-
writer { |io| IOStreams::Line::Writer.stream(io, **args, &block) }
|
192
|
-
end
|
193
|
-
|
194
|
-
def row_writer(delimiter: $/, **args, &block)
|
195
|
-
return block.call(io_stream) if io_stream && io_stream.is_a?(IOStreams::Row::Writer)
|
196
|
-
|
197
|
-
line_writer(delimiter: delimiter) { |io| IOStreams::Row::Writer.stream(io, **args, &block) }
|
198
|
-
end
|
199
|
-
|
200
|
-
def record_writer(delimiter: $/, **args, &block)
|
201
|
-
return block.call(io_stream) if io_stream && io_stream.is_a?(IOStreams::Record::Writer)
|
202
|
-
|
203
|
-
line_writer(delimiter: delimiter) { |io| IOStreams::Record::Writer.stream(io, **args, &block) }
|
204
|
-
end
|
205
|
-
|
206
185
|
# Set/get the original file_name
|
207
186
|
def file_name(file_name = :none)
|
208
187
|
if file_name == :none
|
209
|
-
|
188
|
+
builder.file_name
|
210
189
|
else
|
211
|
-
|
190
|
+
builder.file_name = file_name
|
212
191
|
self
|
213
192
|
end
|
214
193
|
end
|
215
194
|
|
216
195
|
# Set/get the original file_name
|
217
196
|
def file_name=(file_name)
|
218
|
-
|
197
|
+
builder.file_name = file_name
|
219
198
|
end
|
220
199
|
|
221
200
|
# Returns [String] the last component of this path.
|
@@ -230,7 +209,7 @@ module IOStreams
|
|
230
209
|
# IOStreams.path("/home/gumby/work/ruby.rb").basename(".rb") #=> "ruby"
|
231
210
|
# IOStreams.path("/home/gumby/work/ruby.rb").basename(".*") #=> "ruby"
|
232
211
|
def basename(suffix = nil)
|
233
|
-
file_name =
|
212
|
+
file_name = builder.file_name
|
234
213
|
return unless file_name
|
235
214
|
|
236
215
|
suffix.nil? ? ::File.basename(file_name) : ::File.basename(file_name, suffix)
|
@@ -248,7 +227,7 @@ module IOStreams
|
|
248
227
|
# IOStreams.path("test").dirname #=> "."
|
249
228
|
# IOStreams.path(".profile").dirname #=> "."
|
250
229
|
def dirname
|
251
|
-
file_name =
|
230
|
+
file_name = builder.file_name
|
252
231
|
::File.dirname(file_name) if file_name
|
253
232
|
end
|
254
233
|
|
@@ -268,7 +247,7 @@ module IOStreams
|
|
268
247
|
# IOStreams.path(".profile").extname #=> ""
|
269
248
|
# IOStreams.path(".profile.sh").extname #=> ".sh"
|
270
249
|
def extname
|
271
|
-
file_name =
|
250
|
+
file_name = builder.file_name
|
272
251
|
::File.extname(file_name) if file_name
|
273
252
|
end
|
274
253
|
|
@@ -293,8 +272,54 @@ module IOStreams
|
|
293
272
|
|
294
273
|
private
|
295
274
|
|
296
|
-
def
|
297
|
-
@
|
275
|
+
def builder
|
276
|
+
@builder ||= IOStreams::Builder.new
|
277
|
+
end
|
278
|
+
|
279
|
+
def stream_reader(&block)
|
280
|
+
builder.reader(io_stream, &block)
|
281
|
+
end
|
282
|
+
|
283
|
+
def line_reader(embedded_within: nil, **args)
|
284
|
+
embedded_within = '"' if embedded_within.nil? && builder.file_name&.include?('.csv')
|
285
|
+
|
286
|
+
stream_reader { |io| yield IOStreams::Line::Reader.new(io, embedded_within: embedded_within, **args) }
|
287
|
+
end
|
288
|
+
|
289
|
+
# Iterate over a file / stream returning each line as an array, one at a time.
|
290
|
+
def row_reader(delimiter: nil, embedded_within: nil, **args)
|
291
|
+
line_reader(delimiter: delimiter, embedded_within: embedded_within) do |io|
|
292
|
+
yield IOStreams::Row::Reader.new(io, **args)
|
293
|
+
end
|
294
|
+
end
|
295
|
+
|
296
|
+
# Iterate over a file / stream returning each line as a hash, one at a time.
|
297
|
+
def record_reader(delimiter: nil, embedded_within: nil, **args)
|
298
|
+
line_reader(delimiter: delimiter, embedded_within: embedded_within) do |io|
|
299
|
+
yield IOStreams::Record::Reader.new(io, **args)
|
300
|
+
end
|
301
|
+
end
|
302
|
+
|
303
|
+
def stream_writer(&block)
|
304
|
+
builder.writer(io_stream, &block)
|
305
|
+
end
|
306
|
+
|
307
|
+
def line_writer(**args, &block)
|
308
|
+
return block.call(io_stream) if io_stream && io_stream.is_a?(IOStreams::Line::Writer)
|
309
|
+
|
310
|
+
writer { |io| IOStreams::Line::Writer.stream(io, **args, &block) }
|
311
|
+
end
|
312
|
+
|
313
|
+
def row_writer(delimiter: $/, **args, &block)
|
314
|
+
return block.call(io_stream) if io_stream && io_stream.is_a?(IOStreams::Row::Writer)
|
315
|
+
|
316
|
+
line_writer(delimiter: delimiter) { |io| IOStreams::Row::Writer.stream(io, **args, &block) }
|
317
|
+
end
|
318
|
+
|
319
|
+
def record_writer(delimiter: $/, **args, &block)
|
320
|
+
return block.call(io_stream) if io_stream && io_stream.is_a?(IOStreams::Record::Writer)
|
321
|
+
|
322
|
+
line_writer(delimiter: delimiter) { |io| IOStreams::Record::Writer.stream(io, **args, &block) }
|
298
323
|
end
|
299
324
|
end
|
300
325
|
end
|