iostreams 1.0.0.beta7 → 1.0.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +138 -78
- data/lib/io_streams/{streams.rb → builder.rb} +4 -3
- data/lib/io_streams/deprecated.rb +9 -9
- data/lib/io_streams/io_streams.rb +11 -11
- data/lib/io_streams/line/writer.rb +2 -2
- data/lib/io_streams/path.rb +7 -7
- data/lib/io_streams/paths/file.rb +13 -4
- data/lib/io_streams/paths/http.rb +4 -2
- data/lib/io_streams/paths/s3.rb +4 -4
- data/lib/io_streams/paths/sftp.rb +37 -36
- data/lib/io_streams/pgp/reader.rb +3 -1
- data/lib/io_streams/record/writer.rb +1 -1
- data/lib/io_streams/row/writer.rb +1 -1
- data/lib/io_streams/stream.rb +134 -109
- data/lib/io_streams/version.rb +1 -1
- data/lib/iostreams.rb +1 -1
- data/test/{streams_test.rb → builder_test.rb} +23 -11
- data/test/encode_reader_test.rb +1 -0
- data/test/paths/s3_test.rb +1 -1
- data/test/stream_test.rb +26 -26
- metadata +8 -8
@@ -37,15 +37,15 @@ module IOStreams
|
|
37
37
|
# # => "files/file.xls"
|
38
38
|
#
|
39
39
|
# For Files
|
40
|
-
# IOStreams.path('blah.zip').option(:encode, encoding: 'BINARY').
|
41
|
-
# IOStreams.path('blah.zip').option(:encode, encoding: 'UTF-8').
|
42
|
-
# IOStreams.path('blah.zip').option(:encode, encoding: 'UTF-8').
|
43
|
-
# IOStreams.path('blah.zip').option(:encode, encoding: 'UTF-8').
|
40
|
+
# IOStreams.path('blah.zip').option(:encode, encoding: 'BINARY').each(:line) { |line| puts line }
|
41
|
+
# IOStreams.path('blah.zip').option(:encode, encoding: 'UTF-8').each(:line).first
|
42
|
+
# IOStreams.path('blah.zip').option(:encode, encoding: 'UTF-8').each(:hash).last
|
43
|
+
# IOStreams.path('blah.zip').option(:encode, encoding: 'UTF-8').each(:hash).size
|
44
44
|
# IOStreams.path('blah.zip').option(:encode, encoding: 'UTF-8').reader.size
|
45
|
-
# IOStreams.path('blah.csv.zip').
|
46
|
-
# IOStreams.path('blah.zip').option(:pgp, passphrase: 'receiver_passphrase').
|
47
|
-
# IOStreams.path('blah.zip').stream(:zip).stream(:pgp, passphrase: 'receiver_passphrase').
|
48
|
-
# IOStreams.path('blah.zip').stream(:zip).stream(:encode, encoding: 'BINARY').
|
45
|
+
# IOStreams.path('blah.csv.zip').each(:line) { |line| puts line }
|
46
|
+
# IOStreams.path('blah.zip').option(:pgp, passphrase: 'receiver_passphrase').read
|
47
|
+
# IOStreams.path('blah.zip').stream(:zip).stream(:pgp, passphrase: 'receiver_passphrase').read
|
48
|
+
# IOStreams.path('blah.zip').stream(:zip).stream(:encode, encoding: 'BINARY').read
|
49
49
|
#
|
50
50
|
def self.path(*elements, **args)
|
51
51
|
return elements.first if (elements.size == 1) && args.empty? && elements.first.is_a?(IOStreams::Path)
|
@@ -59,9 +59,9 @@ module IOStreams
|
|
59
59
|
|
60
60
|
# For an existing IO Stream
|
61
61
|
# IOStreams.stream(io).file_name('blah.zip').encoding('BINARY').reader(&:read)
|
62
|
-
# IOStreams.stream(io).file_name('blah.zip').encoding('BINARY').
|
63
|
-
# IOStreams.stream(io).file_name('blah.csv.zip').
|
64
|
-
# IOStreams.stream(io).stream(:zip).stream(:pgp, passphrase: 'receiver_passphrase').
|
62
|
+
# IOStreams.stream(io).file_name('blah.zip').encoding('BINARY').each(:line){ ... }
|
63
|
+
# IOStreams.stream(io).file_name('blah.csv.zip').each(:line) { ... }
|
64
|
+
# IOStreams.stream(io).stream(:zip).stream(:pgp, passphrase: 'receiver_passphrase').read
|
65
65
|
def self.stream(io_stream)
|
66
66
|
return io_stream if io_stream.is_a?(Stream)
|
67
67
|
|
@@ -32,7 +32,7 @@ module IOStreams
|
|
32
32
|
# Write a line to the output stream
|
33
33
|
#
|
34
34
|
# Example:
|
35
|
-
# IOStreams.
|
35
|
+
# IOStreams.path('a.txt').writer(:line) do |stream|
|
36
36
|
# stream << 'first line' << 'second line'
|
37
37
|
# end
|
38
38
|
def <<(data)
|
@@ -44,7 +44,7 @@ module IOStreams
|
|
44
44
|
# Returns [Integer] the number of bytes written.
|
45
45
|
#
|
46
46
|
# Example:
|
47
|
-
# IOStreams.
|
47
|
+
# IOStreams.path('a.txt').writer(:line) do |stream|
|
48
48
|
# count = stream.write('first line')
|
49
49
|
# puts "Wrote #{count} bytes to the output file, including the delimiter"
|
50
50
|
# end
|
data/lib/io_streams/path.rb
CHANGED
@@ -8,7 +8,7 @@ module IOStreams
|
|
8
8
|
|
9
9
|
@path = path.frozen? ? path : path.dup.freeze
|
10
10
|
@io_stream = nil
|
11
|
-
@
|
11
|
+
@builder = nil
|
12
12
|
end
|
13
13
|
|
14
14
|
# If elements already contains the current path then it is used as is without
|
@@ -20,7 +20,7 @@ module IOStreams
|
|
20
20
|
relative = ::File.join(*elements)
|
21
21
|
|
22
22
|
new_path = dup
|
23
|
-
new_path.
|
23
|
+
new_path.builder = nil
|
24
24
|
new_path.path = relative.start_with?(path) ? relative : ::File.join(path, relative)
|
25
25
|
new_path
|
26
26
|
end
|
@@ -117,7 +117,7 @@ module IOStreams
|
|
117
117
|
# IOStreams.path(".profile").directory #=> "."
|
118
118
|
def directory
|
119
119
|
new_path = dup
|
120
|
-
new_path.
|
120
|
+
new_path.builder = nil
|
121
121
|
new_path.path = ::File.dirname(path)
|
122
122
|
new_path
|
123
123
|
end
|
@@ -188,15 +188,15 @@ module IOStreams
|
|
188
188
|
|
189
189
|
def inspect
|
190
190
|
str = "#<#{self.class.name}:#{path}"
|
191
|
-
str << " @
|
192
|
-
str << " @options=#{
|
191
|
+
str << " @builder=#{builder.streams.inspect}" if builder.streams
|
192
|
+
str << " @options=#{builder.options.inspect}" if builder.options
|
193
193
|
str << " pipeline=#{pipeline.inspect}>"
|
194
194
|
end
|
195
195
|
|
196
196
|
private
|
197
197
|
|
198
|
-
def
|
199
|
-
@
|
198
|
+
def builder
|
199
|
+
@builder ||= IOStreams::Builder.new(path)
|
200
200
|
end
|
201
201
|
end
|
202
202
|
end
|
@@ -3,6 +3,13 @@ require "fileutils"
|
|
3
3
|
module IOStreams
|
4
4
|
module Paths
|
5
5
|
class File < IOStreams::Path
|
6
|
+
attr_accessor :create_path
|
7
|
+
|
8
|
+
def initialize(file_name, create_path: true)
|
9
|
+
@create_path = create_path
|
10
|
+
super(file_name)
|
11
|
+
end
|
12
|
+
|
6
13
|
# Yields Paths within the current path.
|
7
14
|
#
|
8
15
|
# Examples:
|
@@ -146,9 +153,11 @@ module IOStreams
|
|
146
153
|
self.class.new(::File.realpath(path))
|
147
154
|
end
|
148
155
|
|
156
|
+
private
|
157
|
+
|
149
158
|
# Read from file
|
150
|
-
def
|
151
|
-
::File.open(path, "rb") { |io|
|
159
|
+
def stream_reader(&block)
|
160
|
+
::File.open(path, "rb") { |io| builder.reader(io, &block) }
|
152
161
|
end
|
153
162
|
|
154
163
|
# Write to file
|
@@ -156,10 +165,10 @@ module IOStreams
|
|
156
165
|
# Note:
|
157
166
|
# If an exception is raised whilst the file is being written to the file is removed to
|
158
167
|
# prevent incomplete / partial files from being created.
|
159
|
-
def
|
168
|
+
def stream_writer(&block)
|
160
169
|
mkpath if create_path
|
161
170
|
begin
|
162
|
-
::File.open(path, "wb") { |io|
|
171
|
+
::File.open(path, "wb") { |io| builder.writer(io, &block) }
|
163
172
|
rescue StandardError => e
|
164
173
|
::File.unlink(path) if ::File.exist?(path)
|
165
174
|
raise(e)
|
@@ -47,6 +47,8 @@ module IOStreams
|
|
47
47
|
url
|
48
48
|
end
|
49
49
|
|
50
|
+
private
|
51
|
+
|
50
52
|
# Read a file using an http get.
|
51
53
|
#
|
52
54
|
# For example:
|
@@ -57,7 +59,7 @@ module IOStreams
|
|
57
59
|
#
|
58
60
|
# Notes:
|
59
61
|
# * Since Net::HTTP download only supports a push stream, the data is streamed into a tempfile first.
|
60
|
-
def
|
62
|
+
def stream_reader(&block)
|
61
63
|
handle_redirects(url, http_redirect_count, &block)
|
62
64
|
end
|
63
65
|
|
@@ -91,7 +93,7 @@ module IOStreams
|
|
91
93
|
Utils.temp_file_name('iostreams_http') do |file_name|
|
92
94
|
::File.open(file_name, 'wb') { |io| response.read_body { |chunk| io.write(chunk) } }
|
93
95
|
# Return a read stream
|
94
|
-
result = ::File.open(file_name, "rb") { |io|
|
96
|
+
result = ::File.open(file_name, "rb") { |io| builder.reader(io, &block) }
|
95
97
|
end
|
96
98
|
end
|
97
99
|
end
|
data/lib/io_streams/paths/s3.rb
CHANGED
@@ -208,12 +208,12 @@ module IOStreams
|
|
208
208
|
# TODO: delete_all
|
209
209
|
|
210
210
|
# Read from AWS S3 file.
|
211
|
-
def
|
211
|
+
def stream_reader(&block)
|
212
212
|
# Since S3 download only supports a push stream, write it to a tempfile first.
|
213
213
|
Utils.temp_file_name("iostreams_s3") do |file_name|
|
214
214
|
read_file(file_name)
|
215
215
|
|
216
|
-
::File.open(file_name, "rb") { |io|
|
216
|
+
::File.open(file_name, "rb") { |io| builder.reader(io, &block) }
|
217
217
|
end
|
218
218
|
end
|
219
219
|
|
@@ -231,10 +231,10 @@ module IOStreams
|
|
231
231
|
# aborted and this error is raised. The raised error has a `#errors`
|
232
232
|
# method that returns the failures that caused the upload to be
|
233
233
|
# aborted.
|
234
|
-
def
|
234
|
+
def stream_writer(&block)
|
235
235
|
# Since S3 upload only supports a pull stream, write it to a tempfile first.
|
236
236
|
Utils.temp_file_name("iostreams_s3") do |file_name|
|
237
|
-
result = ::File.open(file_name, "wb") { |io|
|
237
|
+
result = ::File.open(file_name, "wb") { |io| builder.writer(io, &block) }
|
238
238
|
|
239
239
|
# Upload file only once all data has been written to it
|
240
240
|
write_file(file_name)
|
@@ -2,6 +2,26 @@ require 'open3'
|
|
2
2
|
|
3
3
|
module IOStreams
|
4
4
|
module Paths
|
5
|
+
# Read a file from a remote sftp server.
|
6
|
+
#
|
7
|
+
# Example:
|
8
|
+
# IOStreams.
|
9
|
+
# path("sftp://example.org/path/file.txt", username: "jbloggs", password: "secret", compression: false).
|
10
|
+
# reader do |input|
|
11
|
+
# puts input.read
|
12
|
+
# end
|
13
|
+
#
|
14
|
+
# Note:
|
15
|
+
# - raises Net::SFTP::StatusException when the file could not be read.
|
16
|
+
#
|
17
|
+
# Write to a file on a remote sftp server.
|
18
|
+
#
|
19
|
+
# Example:
|
20
|
+
# IOStreams.
|
21
|
+
# path("sftp://example.org/path/file.txt", username: "jbloggs", password: "secret", compression: false).
|
22
|
+
# writer do |output|
|
23
|
+
# output.write('Hello World')
|
24
|
+
# end
|
5
25
|
class SFTP < IOStreams::Path
|
6
26
|
include SemanticLogger::Loggable if defined?(SemanticLogger)
|
7
27
|
|
@@ -54,7 +74,7 @@ module IOStreams
|
|
54
74
|
# IOStreams.path("sftp://test.com/path/file_name.csv", username: "jack", IdentityFile: "~/.ssh/private_key").reader do |io|
|
55
75
|
# puts io.read
|
56
76
|
# end
|
57
|
-
def initialize(url, username: nil, password: nil,
|
77
|
+
def initialize(url, username: nil, password: nil, ssh_options: {})
|
58
78
|
uri = Utils::URI.new(url)
|
59
79
|
raise(ArgumentError, "Invalid URL. Required Format: 'sftp://<host_name>/<file_name>'") unless uri.scheme == 'sftp'
|
60
80
|
|
@@ -87,40 +107,6 @@ module IOStreams
|
|
87
107
|
self
|
88
108
|
end
|
89
109
|
|
90
|
-
# Read a file from a remote sftp server.
|
91
|
-
#
|
92
|
-
# Example:
|
93
|
-
# IOStreams.
|
94
|
-
# path("sftp://example.org/path/file.txt", username: "jbloggs", password: "secret", compression: false).
|
95
|
-
# reader do |input|
|
96
|
-
# puts input.read
|
97
|
-
# end
|
98
|
-
#
|
99
|
-
# Note:
|
100
|
-
# - raises Net::SFTP::StatusException when the file could not be read.
|
101
|
-
def reader(&block)
|
102
|
-
IOStreams.temp_file("iostreams-sftp-reader") do |temp_file|
|
103
|
-
sftp_download(path, temp_file.to_s)
|
104
|
-
::File.open(temp_file.to_s, "rb") { |io| streams.reader(io, &block) }
|
105
|
-
end
|
106
|
-
end
|
107
|
-
|
108
|
-
# Write to a file on a remote sftp server.
|
109
|
-
#
|
110
|
-
# Example:
|
111
|
-
# IOStreams.
|
112
|
-
# path("sftp://example.org/path/file.txt", username: "jbloggs", password: "secret", compression: false).
|
113
|
-
# writer do |output|
|
114
|
-
# output.write('Hello World')
|
115
|
-
# end
|
116
|
-
def writer(&block)
|
117
|
-
IOStreams.temp_file("iostreams-sftp-writer") do |temp_file|
|
118
|
-
::File.open(temp_file.to_s, "wb") { |io| streams.writer(io, &block) }
|
119
|
-
sftp_upload(temp_file.to_s, path)
|
120
|
-
temp_file.size
|
121
|
-
end
|
122
|
-
end
|
123
|
-
|
124
110
|
# TODO: Add #copy_from shortcut to detect when a file is supplied that does not require conversion.
|
125
111
|
|
126
112
|
# Search for files on the remote sftp server that match the provided pattern.
|
@@ -147,7 +133,7 @@ module IOStreams
|
|
147
133
|
Net::SFTP.start(hostname, username, build_ssh_options) do |sftp|
|
148
134
|
sftp.dir.glob(".", pattern, flags) do |path|
|
149
135
|
next if !directories && !path.file?
|
150
|
-
new_path = self.class.new("sftp://#{hostname}/#{path.name}", username: username, password: password,
|
136
|
+
new_path = self.class.new("sftp://#{hostname}/#{path.name}", username: username, password: password, **ssh_options)
|
151
137
|
yield(new_path, path.attributes.attributes)
|
152
138
|
end
|
153
139
|
end
|
@@ -158,6 +144,21 @@ module IOStreams
|
|
158
144
|
|
159
145
|
attr_reader :password
|
160
146
|
|
147
|
+
def stream_reader(&block)
|
148
|
+
IOStreams.temp_file("iostreams-sftp-reader") do |temp_file|
|
149
|
+
sftp_download(path, temp_file.to_s)
|
150
|
+
::File.open(temp_file.to_s, "rb") { |io| builder.reader(io, &block) }
|
151
|
+
end
|
152
|
+
end
|
153
|
+
|
154
|
+
def stream_writer(&block)
|
155
|
+
IOStreams.temp_file("iostreams-sftp-writer") do |temp_file|
|
156
|
+
::File.open(temp_file.to_s, "wb") { |io| builder.writer(io, &block) }
|
157
|
+
sftp_upload(temp_file.to_s, path)
|
158
|
+
temp_file.size
|
159
|
+
end
|
160
|
+
end
|
161
|
+
|
161
162
|
# Use sftp and sshpass executables to download to a local file
|
162
163
|
def sftp_download(remote_file_name, local_file_name)
|
163
164
|
with_sftp_args do |args|
|
@@ -21,7 +21,9 @@ module IOStreams
|
|
21
21
|
#
|
22
22
|
# passphrase: [String]
|
23
23
|
# Pass phrase for private key to decrypt the file with
|
24
|
-
def self.file(file_name, passphrase:
|
24
|
+
def self.file(file_name, passphrase: nil)
|
25
|
+
# Cannot use `passphrase: self.default_passphrase` since it is considered private
|
26
|
+
passphrase ||= default_passphrase
|
25
27
|
raise(ArgumentError, 'Missing both passphrase and IOStreams::Pgp::Reader.default_passphrase') unless passphrase
|
26
28
|
|
27
29
|
loopback = IOStreams::Pgp.pgp_version.to_f >= 2.1 ? '--pinentry-mode loopback' : ''
|
@@ -1,7 +1,7 @@
|
|
1
1
|
module IOStreams
|
2
2
|
module Record
|
3
3
|
# Example, implied header from first record:
|
4
|
-
# IOStreams.path('file.csv').
|
4
|
+
# IOStreams.path('file.csv').writer(:hash) do |stream|
|
5
5
|
# stream << {name: 'Jack', address: 'Somewhere', zipcode: 12345}
|
6
6
|
# stream << {name: 'Joe', address: 'Lost', zipcode: 32443, age: 23}
|
7
7
|
# end
|
@@ -2,7 +2,7 @@ require 'csv'
|
|
2
2
|
module IOStreams
|
3
3
|
module Row
|
4
4
|
# Example:
|
5
|
-
# IOStreams.path("file.csv").
|
5
|
+
# IOStreams.path("file.csv").writer(:array) do |stream|
|
6
6
|
# stream << ['name', 'address', 'zipcode']
|
7
7
|
# stream << ['Jack', 'Somewhere', 12345]
|
8
8
|
# stream << ['Joe', 'Lost', 32443]
|
data/lib/io_streams/stream.rb
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
module IOStreams
|
2
2
|
class Stream
|
3
3
|
attr_reader :io_stream
|
4
|
-
attr_writer :
|
4
|
+
attr_writer :builder
|
5
5
|
|
6
6
|
def initialize(io_stream)
|
7
7
|
raise(ArgumentError, 'io_stream cannot be nil') if io_stream.nil?
|
8
8
|
raise(ArgumentError, "io_stream must not be a string: #{io_stream.inspect}") if io_stream.is_a?(String)
|
9
9
|
|
10
10
|
@io_stream = io_stream
|
11
|
-
@
|
11
|
+
@builder = nil
|
12
12
|
end
|
13
13
|
|
14
14
|
# Ignore the filename and use only the supplied streams.
|
@@ -19,7 +19,7 @@ module IOStreams
|
|
19
19
|
#
|
20
20
|
# IOStreams.path('tempfile2527').stream(:zip).stream(:pgp, passphrase: 'receiver_passphrase').reader(&:read)
|
21
21
|
def stream(stream, **options)
|
22
|
-
|
22
|
+
builder.stream(stream, **options)
|
23
23
|
self
|
24
24
|
end
|
25
25
|
|
@@ -34,32 +34,76 @@ module IOStreams
|
|
34
34
|
#
|
35
35
|
# IOStreams.path(output_file_name).option(:pgp, passphrase: 'receiver_passphrase').reader(&:read)
|
36
36
|
def option(stream, **options)
|
37
|
-
|
37
|
+
builder.option(stream, **options)
|
38
38
|
self
|
39
39
|
end
|
40
40
|
|
41
41
|
# Adds the options for the specified stream as an option,
|
42
42
|
# but if streams have already been added it is instead added as a stream.
|
43
43
|
def option_or_stream(stream, **options)
|
44
|
-
|
44
|
+
builder.option_or_stream(stream, **options)
|
45
45
|
self
|
46
46
|
end
|
47
47
|
|
48
48
|
# Return the options already set for either a stream or option.
|
49
49
|
def setting(stream)
|
50
|
-
|
50
|
+
builder.setting(stream)
|
51
51
|
self
|
52
52
|
end
|
53
53
|
|
54
54
|
# Returns [Hash<Symbol:Hash>] the pipeline of streams
|
55
55
|
# with their options that will be applied when the reader or writer is invoked.
|
56
56
|
def pipeline
|
57
|
-
|
57
|
+
builder.pipeline
|
58
|
+
end
|
59
|
+
|
60
|
+
# Iterate over a file / stream returning one line at a time.
|
61
|
+
#
|
62
|
+
# Example: Read a line at a time
|
63
|
+
# IOStreams.path("file.txt").each(:line) do |line|
|
64
|
+
# puts line
|
65
|
+
# end
|
66
|
+
#
|
67
|
+
# Example: Read a line at a time with custom options
|
68
|
+
# IOStreams.path("file.csv").each(:line, embedded_within: '"') do |line|
|
69
|
+
# puts line
|
70
|
+
# end
|
71
|
+
#
|
72
|
+
# Example: Read a row at a time
|
73
|
+
# IOStreams.path("file.csv").each(:array) do |array|
|
74
|
+
# p array
|
75
|
+
# end
|
76
|
+
#
|
77
|
+
# Example: Read a record at a time
|
78
|
+
# IOStreams.path("file.csv").each(:hash) do |hash|
|
79
|
+
# p hash
|
80
|
+
# end
|
81
|
+
#
|
82
|
+
# Notes:
|
83
|
+
# - Embedded lines (within double quotes) will be skipped if
|
84
|
+
# 1. The file name contains .csv
|
85
|
+
# 2. Or the embedded_within argument is set
|
86
|
+
def each(mode = :line, **args, &block)
|
87
|
+
raise(ArgumentError, "Invalid mode: #{mode.inspect}") if mode == :stream
|
88
|
+
|
89
|
+
# return enum_for __method__ unless block_given?
|
90
|
+
reader(mode, **args) { |stream| stream.each(&block) }
|
58
91
|
end
|
59
92
|
|
60
93
|
# Returns a Reader for reading a file / stream
|
61
|
-
def reader(&block)
|
62
|
-
|
94
|
+
def reader(mode = :stream, **args, &block)
|
95
|
+
case mode
|
96
|
+
when :stream
|
97
|
+
stream_reader(&block)
|
98
|
+
when :line
|
99
|
+
line_reader(**args, &block)
|
100
|
+
when :array
|
101
|
+
row_reader(**args, &block)
|
102
|
+
when :hash
|
103
|
+
record_reader(**args, &block)
|
104
|
+
else
|
105
|
+
raise(ArgumentError, "Invalid mode: #{mode.inspect}")
|
106
|
+
end
|
63
107
|
end
|
64
108
|
|
65
109
|
# Read an entire file into memory.
|
@@ -67,12 +111,37 @@ module IOStreams
|
|
67
111
|
# Notes:
|
68
112
|
# - Use with caution since large files can cause a denial of service since
|
69
113
|
# this method will load the entire file into memory.
|
70
|
-
# - Recommend using instead `#reader
|
71
|
-
# block into memory at a time.
|
114
|
+
# - Recommend using instead `#reader` to read a block into memory at a time.
|
72
115
|
def read(*args)
|
73
116
|
reader { |stream| stream.read(*args) }
|
74
117
|
end
|
75
118
|
|
119
|
+
# Returns a Writer for writing to a file / stream
|
120
|
+
def writer(mode = :stream, **args, &block)
|
121
|
+
case mode
|
122
|
+
when :stream
|
123
|
+
stream_writer(&block)
|
124
|
+
when :line
|
125
|
+
line_writer(**args, &block)
|
126
|
+
when :array
|
127
|
+
row_writer(**args, &block)
|
128
|
+
when :hash
|
129
|
+
record_writer(**args, &block)
|
130
|
+
else
|
131
|
+
raise(ArgumentError, "Invalid mode: #{mode.inspect}")
|
132
|
+
end
|
133
|
+
end
|
134
|
+
|
135
|
+
# Write entire string to file.
|
136
|
+
#
|
137
|
+
# Notes:
|
138
|
+
# - Use with caution since preparing large amounts of data in memory can cause a denial of service
|
139
|
+
# since all the data for the file needs to be resident in memory before writing.
|
140
|
+
# - Recommend using instead `#writer` to write a block of memory at a time.
|
141
|
+
def write(data)
|
142
|
+
writer { |stream| stream.write(data) }
|
143
|
+
end
|
144
|
+
|
76
145
|
# Copy from another stream, path, file_name or IO instance.
|
77
146
|
#
|
78
147
|
# Parameters:
|
@@ -113,109 +182,19 @@ module IOStreams
|
|
113
182
|
target.copy_from(self, convert: convert)
|
114
183
|
end
|
115
184
|
|
116
|
-
# Iterate over a file / stream returning one line at a time.
|
117
|
-
# Embedded lines (within double quotes) will be skipped if
|
118
|
-
# 1. The file name contains .csv
|
119
|
-
# 2. Or the embedded_within argument is set
|
120
|
-
#
|
121
|
-
# Example: Supply custom options
|
122
|
-
# IOStreams.each_line(file_name, embedded_within: '"') do |line|
|
123
|
-
# puts line
|
124
|
-
# end
|
125
|
-
#
|
126
|
-
def each_line(**args, &block)
|
127
|
-
# return enum_for __method__ unless block_given?
|
128
|
-
line_reader(**args) { |line_stream| line_stream.each(&block) }
|
129
|
-
end
|
130
|
-
|
131
|
-
# Iterate over a file / stream returning one line at a time.
|
132
|
-
# Embedded lines (within double quotes) will be skipped if
|
133
|
-
# 1. The file name contains .csv
|
134
|
-
# 2. Or the embedded_within argument is set
|
135
|
-
#
|
136
|
-
# Example: Supply custom options
|
137
|
-
# IOStreams.each_row(file_name, embedded_within: '"') do |line|
|
138
|
-
# puts line
|
139
|
-
# end
|
140
|
-
#
|
141
|
-
def each_row(**args, &block)
|
142
|
-
row_reader(**args) { |row_stream| row_stream.each(&block) }
|
143
|
-
end
|
144
|
-
|
145
|
-
# Returns [Hash] of every record in a file or stream with support for headers.
|
146
|
-
def each_record(**args, &block)
|
147
|
-
record_reader(**args) { |record_stream| record_stream.each(&block) }
|
148
|
-
end
|
149
|
-
|
150
|
-
# Iterate over a file / stream returning each record/line one at a time.
|
151
|
-
# It will apply the embedded_within argument if the file or input_stream contain .csv in its name.
|
152
|
-
def line_reader(embedded_within: nil, **args)
|
153
|
-
embedded_within = '"' if embedded_within.nil? && streams.file_name&.include?('.csv')
|
154
|
-
|
155
|
-
reader { |io| yield IOStreams::Line::Reader.new(io, embedded_within: embedded_within, **args) }
|
156
|
-
end
|
157
|
-
|
158
|
-
# Iterate over a file / stream returning each line as an array, one at a time.
|
159
|
-
def row_reader(delimiter: nil, embedded_within: nil, **args)
|
160
|
-
line_reader(delimiter: delimiter, embedded_within: embedded_within) do |io|
|
161
|
-
yield IOStreams::Row::Reader.new(io, **args)
|
162
|
-
end
|
163
|
-
end
|
164
|
-
|
165
|
-
# Iterate over a file / stream returning each line as a hash, one at a time.
|
166
|
-
def record_reader(delimiter: nil, embedded_within: nil, **args)
|
167
|
-
line_reader(delimiter: delimiter, embedded_within: embedded_within) do |io|
|
168
|
-
yield IOStreams::Record::Reader.new(io, **args)
|
169
|
-
end
|
170
|
-
end
|
171
|
-
|
172
|
-
# Returns a Writer for writing to a file / stream
|
173
|
-
def writer(&block)
|
174
|
-
streams.writer(io_stream, &block)
|
175
|
-
end
|
176
|
-
|
177
|
-
# Write entire string to file.
|
178
|
-
#
|
179
|
-
# Notes:
|
180
|
-
# - Use with caution since preparing large amounts of data in memory can cause a denial of service
|
181
|
-
# since all the data for the file needs to be resident in memory before writing.
|
182
|
-
# - Recommend using instead `#writer`, `#line_writer`, or `#row_writer` to write a
|
183
|
-
# block of memory at a time.
|
184
|
-
def write(data)
|
185
|
-
writer { |stream| stream.write(data) }
|
186
|
-
end
|
187
|
-
|
188
|
-
def line_writer(**args, &block)
|
189
|
-
return block.call(io_stream) if io_stream && io_stream.is_a?(IOStreams::Line::Writer)
|
190
|
-
|
191
|
-
writer { |io| IOStreams::Line::Writer.stream(io, **args, &block) }
|
192
|
-
end
|
193
|
-
|
194
|
-
def row_writer(delimiter: $/, **args, &block)
|
195
|
-
return block.call(io_stream) if io_stream && io_stream.is_a?(IOStreams::Row::Writer)
|
196
|
-
|
197
|
-
line_writer(delimiter: delimiter) { |io| IOStreams::Row::Writer.stream(io, **args, &block) }
|
198
|
-
end
|
199
|
-
|
200
|
-
def record_writer(delimiter: $/, **args, &block)
|
201
|
-
return block.call(io_stream) if io_stream && io_stream.is_a?(IOStreams::Record::Writer)
|
202
|
-
|
203
|
-
line_writer(delimiter: delimiter) { |io| IOStreams::Record::Writer.stream(io, **args, &block) }
|
204
|
-
end
|
205
|
-
|
206
185
|
# Set/get the original file_name
|
207
186
|
def file_name(file_name = :none)
|
208
187
|
if file_name == :none
|
209
|
-
|
188
|
+
builder.file_name
|
210
189
|
else
|
211
|
-
|
190
|
+
builder.file_name = file_name
|
212
191
|
self
|
213
192
|
end
|
214
193
|
end
|
215
194
|
|
216
195
|
# Set/get the original file_name
|
217
196
|
def file_name=(file_name)
|
218
|
-
|
197
|
+
builder.file_name = file_name
|
219
198
|
end
|
220
199
|
|
221
200
|
# Returns [String] the last component of this path.
|
@@ -230,7 +209,7 @@ module IOStreams
|
|
230
209
|
# IOStreams.path("/home/gumby/work/ruby.rb").basename(".rb") #=> "ruby"
|
231
210
|
# IOStreams.path("/home/gumby/work/ruby.rb").basename(".*") #=> "ruby"
|
232
211
|
def basename(suffix = nil)
|
233
|
-
file_name =
|
212
|
+
file_name = builder.file_name
|
234
213
|
return unless file_name
|
235
214
|
|
236
215
|
suffix.nil? ? ::File.basename(file_name) : ::File.basename(file_name, suffix)
|
@@ -248,7 +227,7 @@ module IOStreams
|
|
248
227
|
# IOStreams.path("test").dirname #=> "."
|
249
228
|
# IOStreams.path(".profile").dirname #=> "."
|
250
229
|
def dirname
|
251
|
-
file_name =
|
230
|
+
file_name = builder.file_name
|
252
231
|
::File.dirname(file_name) if file_name
|
253
232
|
end
|
254
233
|
|
@@ -268,7 +247,7 @@ module IOStreams
|
|
268
247
|
# IOStreams.path(".profile").extname #=> ""
|
269
248
|
# IOStreams.path(".profile.sh").extname #=> ".sh"
|
270
249
|
def extname
|
271
|
-
file_name =
|
250
|
+
file_name = builder.file_name
|
272
251
|
::File.extname(file_name) if file_name
|
273
252
|
end
|
274
253
|
|
@@ -293,8 +272,54 @@ module IOStreams
|
|
293
272
|
|
294
273
|
private
|
295
274
|
|
296
|
-
def
|
297
|
-
@
|
275
|
+
def builder
|
276
|
+
@builder ||= IOStreams::Builder.new
|
277
|
+
end
|
278
|
+
|
279
|
+
def stream_reader(&block)
|
280
|
+
builder.reader(io_stream, &block)
|
281
|
+
end
|
282
|
+
|
283
|
+
def line_reader(embedded_within: nil, **args)
|
284
|
+
embedded_within = '"' if embedded_within.nil? && builder.file_name&.include?('.csv')
|
285
|
+
|
286
|
+
stream_reader { |io| yield IOStreams::Line::Reader.new(io, embedded_within: embedded_within, **args) }
|
287
|
+
end
|
288
|
+
|
289
|
+
# Iterate over a file / stream returning each line as an array, one at a time.
|
290
|
+
def row_reader(delimiter: nil, embedded_within: nil, **args)
|
291
|
+
line_reader(delimiter: delimiter, embedded_within: embedded_within) do |io|
|
292
|
+
yield IOStreams::Row::Reader.new(io, **args)
|
293
|
+
end
|
294
|
+
end
|
295
|
+
|
296
|
+
# Iterate over a file / stream returning each line as a hash, one at a time.
|
297
|
+
def record_reader(delimiter: nil, embedded_within: nil, **args)
|
298
|
+
line_reader(delimiter: delimiter, embedded_within: embedded_within) do |io|
|
299
|
+
yield IOStreams::Record::Reader.new(io, **args)
|
300
|
+
end
|
301
|
+
end
|
302
|
+
|
303
|
+
def stream_writer(&block)
|
304
|
+
builder.writer(io_stream, &block)
|
305
|
+
end
|
306
|
+
|
307
|
+
def line_writer(**args, &block)
|
308
|
+
return block.call(io_stream) if io_stream && io_stream.is_a?(IOStreams::Line::Writer)
|
309
|
+
|
310
|
+
writer { |io| IOStreams::Line::Writer.stream(io, **args, &block) }
|
311
|
+
end
|
312
|
+
|
313
|
+
def row_writer(delimiter: $/, **args, &block)
|
314
|
+
return block.call(io_stream) if io_stream && io_stream.is_a?(IOStreams::Row::Writer)
|
315
|
+
|
316
|
+
line_writer(delimiter: delimiter) { |io| IOStreams::Row::Writer.stream(io, **args, &block) }
|
317
|
+
end
|
318
|
+
|
319
|
+
def record_writer(delimiter: $/, **args, &block)
|
320
|
+
return block.call(io_stream) if io_stream && io_stream.is_a?(IOStreams::Record::Writer)
|
321
|
+
|
322
|
+
line_writer(delimiter: delimiter) { |io| IOStreams::Record::Writer.stream(io, **args, &block) }
|
298
323
|
end
|
299
324
|
end
|
300
325
|
end
|