iostreams 0.20.3 → 1.0.0.beta
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/io_streams/bzip2/reader.rb +9 -21
- data/lib/io_streams/bzip2/writer.rb +9 -21
- data/lib/io_streams/deprecated.rb +217 -0
- data/lib/io_streams/encode/reader.rb +12 -16
- data/lib/io_streams/encode/writer.rb +9 -13
- data/lib/io_streams/errors.rb +6 -6
- data/lib/io_streams/gzip/reader.rb +7 -14
- data/lib/io_streams/gzip/writer.rb +7 -15
- data/lib/io_streams/io_streams.rb +182 -524
- data/lib/io_streams/line/reader.rb +9 -9
- data/lib/io_streams/line/writer.rb +10 -11
- data/lib/io_streams/path.rb +190 -0
- data/lib/io_streams/paths/file.rb +176 -0
- data/lib/io_streams/paths/http.rb +92 -0
- data/lib/io_streams/paths/matcher.rb +61 -0
- data/lib/io_streams/paths/s3.rb +269 -0
- data/lib/io_streams/paths/sftp.rb +99 -0
- data/lib/io_streams/pgp.rb +47 -19
- data/lib/io_streams/pgp/reader.rb +20 -28
- data/lib/io_streams/pgp/writer.rb +24 -46
- data/lib/io_streams/reader.rb +28 -0
- data/lib/io_streams/record/reader.rb +20 -16
- data/lib/io_streams/record/writer.rb +28 -28
- data/lib/io_streams/row/reader.rb +22 -26
- data/lib/io_streams/row/writer.rb +29 -28
- data/lib/io_streams/stream.rb +400 -0
- data/lib/io_streams/streams.rb +125 -0
- data/lib/io_streams/symmetric_encryption/reader.rb +5 -13
- data/lib/io_streams/symmetric_encryption/writer.rb +16 -15
- data/lib/io_streams/tabular/header.rb +9 -3
- data/lib/io_streams/tabular/parser/array.rb +8 -3
- data/lib/io_streams/tabular/parser/csv.rb +6 -2
- data/lib/io_streams/tabular/parser/hash.rb +4 -1
- data/lib/io_streams/tabular/parser/json.rb +3 -1
- data/lib/io_streams/tabular/parser/psv.rb +3 -1
- data/lib/io_streams/tabular/utility/csv_row.rb +9 -8
- data/lib/io_streams/utils.rb +22 -0
- data/lib/io_streams/version.rb +1 -1
- data/lib/io_streams/writer.rb +28 -0
- data/lib/io_streams/xlsx/reader.rb +7 -19
- data/lib/io_streams/zip/reader.rb +7 -26
- data/lib/io_streams/zip/writer.rb +21 -38
- data/lib/iostreams.rb +15 -15
- data/test/bzip2_reader_test.rb +3 -3
- data/test/bzip2_writer_test.rb +3 -3
- data/test/deprecated_test.rb +123 -0
- data/test/encode_reader_test.rb +3 -3
- data/test/encode_writer_test.rb +6 -6
- data/test/gzip_reader_test.rb +2 -2
- data/test/gzip_writer_test.rb +3 -3
- data/test/io_streams_test.rb +43 -136
- data/test/line_reader_test.rb +20 -20
- data/test/line_writer_test.rb +3 -3
- data/test/path_test.rb +30 -28
- data/test/paths/file_test.rb +206 -0
- data/test/paths/http_test.rb +34 -0
- data/test/paths/matcher_test.rb +111 -0
- data/test/paths/s3_test.rb +207 -0
- data/test/pgp_reader_test.rb +8 -8
- data/test/pgp_writer_test.rb +13 -13
- data/test/record_reader_test.rb +5 -5
- data/test/record_writer_test.rb +4 -4
- data/test/row_reader_test.rb +5 -5
- data/test/row_writer_test.rb +6 -6
- data/test/stream_test.rb +116 -0
- data/test/streams_test.rb +255 -0
- data/test/utils_test.rb +20 -0
- data/test/xlsx_reader_test.rb +3 -3
- data/test/zip_reader_test.rb +12 -12
- data/test/zip_writer_test.rb +5 -5
- metadata +33 -45
- data/lib/io_streams/base_path.rb +0 -72
- data/lib/io_streams/file/path.rb +0 -58
- data/lib/io_streams/file/reader.rb +0 -12
- data/lib/io_streams/file/writer.rb +0 -22
- data/lib/io_streams/http/reader.rb +0 -71
- data/lib/io_streams/s3.rb +0 -26
- data/lib/io_streams/s3/path.rb +0 -40
- data/lib/io_streams/s3/reader.rb +0 -28
- data/lib/io_streams/s3/writer.rb +0 -85
- data/lib/io_streams/sftp/reader.rb +0 -67
- data/lib/io_streams/sftp/writer.rb +0 -68
- data/test/base_path_test.rb +0 -35
- data/test/file_path_test.rb +0 -97
- data/test/file_reader_test.rb +0 -33
- data/test/file_writer_test.rb +0 -50
- data/test/http_reader_test.rb +0 -38
- data/test/s3_reader_test.rb +0 -41
- data/test/s3_writer_test.rb +0 -41
@@ -1,6 +1,6 @@
|
|
1
1
|
module IOStreams
|
2
2
|
module Line
|
3
|
-
class Reader
|
3
|
+
class Reader < IOStreams::Reader
|
4
4
|
attr_reader :delimiter, :buffer_size, :line_number
|
5
5
|
|
6
6
|
# Prevent denial of service when a delimiter is not found before this number * `buffer_size` characters are read.
|
@@ -8,13 +8,12 @@ module IOStreams
|
|
8
8
|
|
9
9
|
LINEFEED_REGEXP = Regexp.compile(/\r\n|\n|\r/).freeze
|
10
10
|
|
11
|
-
# Read a line at a time from a
|
12
|
-
def self.
|
13
|
-
if
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
end
|
11
|
+
# Read a line at a time from a stream
|
12
|
+
def self.stream(input_stream, original_file_name: nil, **args, &block)
|
13
|
+
# Pass-through if already a line reader
|
14
|
+
return block.call(input_stream) if input_stream.is_a?(self.class)
|
15
|
+
|
16
|
+
yield new(input_stream, **args)
|
18
17
|
end
|
19
18
|
|
20
19
|
# Create a delimited stream reader from the supplied input stream.
|
@@ -46,8 +45,9 @@ module IOStreams
|
|
46
45
|
# - Extract header line(s) / first non-comment, non-blank line
|
47
46
|
# - Embedded newline support, RegExp? or Proc?
|
48
47
|
def initialize(input_stream, delimiter: nil, buffer_size: 65_536, embedded_within: nil)
|
48
|
+
super(input_stream)
|
49
|
+
|
49
50
|
@embedded_within = embedded_within
|
50
|
-
@input_stream = input_stream
|
51
51
|
@buffer_size = buffer_size
|
52
52
|
|
53
53
|
# More efficient read buffering only supported when the input stream `#read` method supports it.
|
@@ -1,15 +1,14 @@
|
|
1
1
|
module IOStreams
|
2
2
|
module Line
|
3
|
-
class Writer
|
3
|
+
class Writer < IOStreams::Writer
|
4
4
|
attr_reader :delimiter
|
5
5
|
|
6
|
-
# Write a line at a time to a
|
7
|
-
def self.
|
8
|
-
if
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
end
|
6
|
+
# Write a line at a time to a stream.
|
7
|
+
def self.stream(output_stream, original_file_name: nil, **args, &block)
|
8
|
+
# Pass-through if already a line writer
|
9
|
+
return block.call(output_stream) if output_stream.is_a?(self.class)
|
10
|
+
|
11
|
+
yield new(output_stream, **args)
|
13
12
|
end
|
14
13
|
|
15
14
|
# A delimited stream writer that will write to the supplied output stream.
|
@@ -26,8 +25,8 @@ module IOStreams
|
|
26
25
|
# to the output stream
|
27
26
|
# Default: OS Specific. Linux: "\n"
|
28
27
|
def initialize(output_stream, delimiter: $/)
|
29
|
-
|
30
|
-
@delimiter
|
28
|
+
super(output_stream)
|
29
|
+
@delimiter = delimiter
|
31
30
|
end
|
32
31
|
|
33
32
|
# Write a line to the output stream
|
@@ -50,7 +49,7 @@ module IOStreams
|
|
50
49
|
# puts "Wrote #{count} bytes to the output file, including the delimiter"
|
51
50
|
# end
|
52
51
|
def write(data)
|
53
|
-
|
52
|
+
output_stream.write(data.to_s + delimiter)
|
54
53
|
end
|
55
54
|
end
|
56
55
|
end
|
@@ -0,0 +1,190 @@
|
|
1
|
+
module IOStreams
|
2
|
+
class Path < IOStreams::Stream
|
3
|
+
attr_reader :path
|
4
|
+
|
5
|
+
def initialize(path)
|
6
|
+
raise(ArgumentError, 'Path cannot be nil') if path.nil?
|
7
|
+
raise(ArgumentError, "Path must be a string: #{path.inspect}, class: #{path.class}") unless path.is_a?(String)
|
8
|
+
|
9
|
+
@path = path.frozen? ? path : path.dup.freeze
|
10
|
+
@io_stream = nil
|
11
|
+
@streams = nil
|
12
|
+
end
|
13
|
+
|
14
|
+
# If elements already contains the current path then it is used as is without
|
15
|
+
# adding the current path for a second time
|
16
|
+
def join(*elements)
|
17
|
+
return self if elements.empty?
|
18
|
+
|
19
|
+
elements = elements.collect(&:to_s)
|
20
|
+
relative = ::File.join(*elements)
|
21
|
+
if relative.start_with?(path)
|
22
|
+
self.class.new(relative)
|
23
|
+
else
|
24
|
+
self.class.new(::File.join(path, relative))
|
25
|
+
end
|
26
|
+
end
|
27
|
+
|
28
|
+
def relative?
|
29
|
+
!absolute?
|
30
|
+
end
|
31
|
+
|
32
|
+
def absolute?
|
33
|
+
!!(path.strip =~ /\A\//)
|
34
|
+
end
|
35
|
+
|
36
|
+
# By default realpath just returns self.
|
37
|
+
def realpath
|
38
|
+
self
|
39
|
+
end
|
40
|
+
|
41
|
+
# Runs the pattern from the current path, returning the complete path for located files.
|
42
|
+
#
|
43
|
+
# See IOStreams::Paths::File.each for arguments.
|
44
|
+
def each_child(pattern = "*", **args, &block)
|
45
|
+
raise NotImplementedError
|
46
|
+
end
|
47
|
+
|
48
|
+
# Returns [Array] of child files based on the supplied pattern
|
49
|
+
def children(*args, **kargs)
|
50
|
+
paths = []
|
51
|
+
each_child(*args, **kargs) { |path| paths << path }
|
52
|
+
paths
|
53
|
+
end
|
54
|
+
|
55
|
+
# Returns [String] the current path.
|
56
|
+
def to_s
|
57
|
+
path
|
58
|
+
end
|
59
|
+
|
60
|
+
# Removes the last element of the path, the file name, before creating the entire path.
|
61
|
+
# Returns self
|
62
|
+
def mkpath
|
63
|
+
raise NotImplementedError
|
64
|
+
end
|
65
|
+
|
66
|
+
# Assumes the current path does not include a file name, and creates all elements in the path.
|
67
|
+
# Returns self
|
68
|
+
#
|
69
|
+
# Note: Do not call this method if the path contains a file name, see `#mkpath`
|
70
|
+
def mkdir
|
71
|
+
raise NotImplementedError
|
72
|
+
end
|
73
|
+
|
74
|
+
# Returns [true|false] whether the file exists
|
75
|
+
def exist?
|
76
|
+
raise NotImplementedError
|
77
|
+
end
|
78
|
+
|
79
|
+
# Returns [Integer] size of the file
|
80
|
+
def size
|
81
|
+
raise NotImplementedError
|
82
|
+
end
|
83
|
+
|
84
|
+
# Cleanup an incomplete write to the target "file" if the copy fails.
|
85
|
+
def copy_from(source, **args)
|
86
|
+
super(source, **args)
|
87
|
+
rescue StandardError => exc
|
88
|
+
delete
|
89
|
+
raise(exc)
|
90
|
+
end
|
91
|
+
|
92
|
+
# Moves the file by copying it to the new path and then deleting the current path.
|
93
|
+
# Returns [IOStreams::Path] the target path.
|
94
|
+
#
|
95
|
+
# Notes:
|
96
|
+
# - Currently only supports moving individual files, not directories.
|
97
|
+
def move_to(target_path)
|
98
|
+
target = IOStreams.new(target_path)
|
99
|
+
target.mkpath
|
100
|
+
target.copy_from(self, convert: false)
|
101
|
+
delete
|
102
|
+
target
|
103
|
+
end
|
104
|
+
|
105
|
+
# Returns [IOStreams::Path] the directory for this file.
|
106
|
+
# Returns `nil` if no `file_name` was set.
|
107
|
+
#
|
108
|
+
# If `path` does not include a directory name then "." is returned.
|
109
|
+
#
|
110
|
+
# IOStreams.path("test.rb").directory #=> "."
|
111
|
+
# IOStreams.path("a/b/d/test.rb").directory #=> "a/b/d"
|
112
|
+
# IOStreams.path(".a/b/d/test.rb").directory #=> ".a/b/d"
|
113
|
+
# IOStreams.path("foo.").directory #=> "."
|
114
|
+
# IOStreams.path("test").directory #=> "."
|
115
|
+
# IOStreams.path(".profile").directory #=> "."
|
116
|
+
def directory
|
117
|
+
file_name = streams.file_name
|
118
|
+
self.class.new(::File.dirname(file_name)) if file_name
|
119
|
+
end
|
120
|
+
|
121
|
+
# When path is a file, deletes this file.
|
122
|
+
# When path is a directory, attempts to delete this directory. If the directory contains
|
123
|
+
# any children it will fail.
|
124
|
+
#
|
125
|
+
# Returns self
|
126
|
+
#
|
127
|
+
# Notes:
|
128
|
+
# * No error is raised if the file or directory is not present.
|
129
|
+
# * Only the file is removed, not any of the parent paths.
|
130
|
+
def delete
|
131
|
+
raise NotImplementedError
|
132
|
+
end
|
133
|
+
|
134
|
+
# When path is a directory ,deletes this directory and all its children.
|
135
|
+
# When path is a file ,deletes this file.
|
136
|
+
#
|
137
|
+
# Returns self
|
138
|
+
#
|
139
|
+
# Notes:
|
140
|
+
# * No error is raised if the file is not present.
|
141
|
+
# * Only the file is removed, not any of the parent paths.
|
142
|
+
# * All children paths and files will be removed.
|
143
|
+
def delete_all
|
144
|
+
raise NotImplementedError
|
145
|
+
end
|
146
|
+
|
147
|
+
# Returns [true|false] whether the file is compressed based on its file extensions.
|
148
|
+
def compressed?
|
149
|
+
# TODO: Look at streams?
|
150
|
+
!(path =~ /\.(zip|gz|gzip|xls.|)\z/i).nil?
|
151
|
+
end
|
152
|
+
|
153
|
+
# Returns [true|false] whether the file is encrypted based on its file extensions.
|
154
|
+
def encrypted?
|
155
|
+
# TODO: Look at streams?
|
156
|
+
!(path =~ /\.(enc|pgp|gpg)\z/i).nil?
|
157
|
+
end
|
158
|
+
|
159
|
+
# TODO: Other possible methods:
|
160
|
+
# - rename - File.rename
|
161
|
+
# - rmtree - delete everything under this path - FileUtils.rm_r
|
162
|
+
# - directory?
|
163
|
+
# - file?
|
164
|
+
# - empty?
|
165
|
+
# - find(ignore_error: true) - Find.find
|
166
|
+
|
167
|
+
# Paths are sortable by name
|
168
|
+
def <=>(other)
|
169
|
+
path <=> other.to_s
|
170
|
+
end
|
171
|
+
|
172
|
+
# Compare by path name, ignore streams
|
173
|
+
def ==(other)
|
174
|
+
path == other.to_s
|
175
|
+
end
|
176
|
+
|
177
|
+
def inspect
|
178
|
+
str = "#<#{self.class.name}:#{path}"
|
179
|
+
str << " @streams=#{streams.streams.inspect}" if streams.streams
|
180
|
+
str << " @options=#{streams.options.inspect}" if streams.options
|
181
|
+
str << " pipeline=#{pipeline.inspect}>"
|
182
|
+
end
|
183
|
+
|
184
|
+
private
|
185
|
+
|
186
|
+
def streams
|
187
|
+
@streams ||= IOStreams::Streams.new(path)
|
188
|
+
end
|
189
|
+
end
|
190
|
+
end
|
@@ -0,0 +1,176 @@
|
|
1
|
+
require "fileutils"
|
2
|
+
|
3
|
+
module IOStreams
|
4
|
+
module Paths
|
5
|
+
class File < IOStreams::Path
|
6
|
+
# Returns a path to a temporary file.
|
7
|
+
# Temporary file is deleted upon block completion if present.
|
8
|
+
def self.temp_file(basename, extension = "")
|
9
|
+
Utils.temp_file_name(basename, extension) { |file_name| yield(new(file_name).stream(:none)) }
|
10
|
+
end
|
11
|
+
|
12
|
+
# Yields Paths within the current path.
|
13
|
+
#
|
14
|
+
# Examples:
|
15
|
+
#
|
16
|
+
# # Case Insensitive file name lookup:
|
17
|
+
# IOStreams::Paths::File.new("ruby").glob("r*.md") { |name| puts name }
|
18
|
+
#
|
19
|
+
# # Case Sensitive file name lookup:
|
20
|
+
# IOStreams::Paths::File.new("ruby").each("R*.md", case_sensitive: true) { |name| puts name }
|
21
|
+
#
|
22
|
+
# # Also return the names of directories found during the search:
|
23
|
+
# IOStreams::Paths::File.new("ruby").each("R*.md", directories: true) { |name| puts name }
|
24
|
+
#
|
25
|
+
# # Case Insensitive recursive file name lookup:
|
26
|
+
# IOStreams::Paths::File.new("ruby").glob("**/*.md") { |name| puts name }
|
27
|
+
#
|
28
|
+
# Parameters:
|
29
|
+
# pattern [String]
|
30
|
+
# The pattern is not a regexp, it is a string that may contain the following metacharacters:
|
31
|
+
# `*` Matches all regular files.
|
32
|
+
# `c*` Matches all regular files beginning with `c`.
|
33
|
+
# `*c` Matches all regular files ending with `c`.
|
34
|
+
# `*c*` Matches all regular files that have `c` in them.
|
35
|
+
#
|
36
|
+
# `**` Matches recursively into subdirectories.
|
37
|
+
#
|
38
|
+
# `?` Matches any one character.
|
39
|
+
#
|
40
|
+
# `[set]` Matches any one character in the supplied `set`.
|
41
|
+
# `[^set]` Does not matches any one character in the supplied `set`.
|
42
|
+
#
|
43
|
+
# `\` Escapes the next metacharacter.
|
44
|
+
#
|
45
|
+
# `{a,b}` Matches on either pattern `a` or pattern `b`.
|
46
|
+
#
|
47
|
+
# case_sensitive [true|false]
|
48
|
+
# Whether the pattern is case-sensitive.
|
49
|
+
#
|
50
|
+
# directories [true|false]
|
51
|
+
# Whether to yield directory names.
|
52
|
+
#
|
53
|
+
# hidden [true|false]
|
54
|
+
# Whether to yield hidden paths.
|
55
|
+
#
|
56
|
+
# Examples:
|
57
|
+
#
|
58
|
+
# Pattern: File name: match? Reason Options
|
59
|
+
# =========== ================ ====== ============================= ===========================
|
60
|
+
# "cat" "cat" true # Match entire string
|
61
|
+
# "cat" "category" false # Only match partial string
|
62
|
+
#
|
63
|
+
# "c{at,ub}s" "cats" true # { } is supported
|
64
|
+
#
|
65
|
+
# "c?t" "cat" true # "?" match only 1 character
|
66
|
+
# "c??t" "cat" false # ditto
|
67
|
+
# "c*" "cats" true # "*" match 0 or more characters
|
68
|
+
# "c*t" "c/a/b/t" true # ditto
|
69
|
+
# "ca[a-z]" "cat" true # inclusive bracket expression
|
70
|
+
# "ca[^t]" "cat" false # exclusive bracket expression ("^" or "!")
|
71
|
+
#
|
72
|
+
# "cat" "CAT" false # case sensitive {case_sensitive: false}
|
73
|
+
# "cat" "CAT" true # case insensitive
|
74
|
+
#
|
75
|
+
# "\?" "?" true # escaped wildcard becomes ordinary
|
76
|
+
# "\a" "a" true # escaped ordinary remains ordinary
|
77
|
+
# "[\?]" "?" true # can escape inside bracket expression
|
78
|
+
#
|
79
|
+
# "*" ".profile" false # wildcard doesn't match leading
|
80
|
+
# "*" ".profile" true # period by default.
|
81
|
+
# ".*" ".profile" true {hidden: true}
|
82
|
+
#
|
83
|
+
# "**/*.rb" "main.rb" false
|
84
|
+
# "**/*.rb" "./main.rb" false
|
85
|
+
# "**/*.rb" "lib/song.rb" true
|
86
|
+
# "**.rb" "main.rb" true
|
87
|
+
# "**.rb" "./main.rb" false
|
88
|
+
# "**.rb" "lib/song.rb" true
|
89
|
+
# "*" "dave/.profile" true
|
90
|
+
def each_child(pattern = "*", case_sensitive: false, directories: false, hidden: false)
|
91
|
+
flags = 0
|
92
|
+
flags |= ::File::FNM_CASEFOLD unless case_sensitive
|
93
|
+
flags |= ::File::FNM_DOTMATCH if hidden
|
94
|
+
|
95
|
+
# Dir.each_child("testdir") {|x| puts "Got #{x}" }
|
96
|
+
Dir.glob(::File.join(path, pattern), flags) do |full_path|
|
97
|
+
next if !directories && ::File.directory?(full_path)
|
98
|
+
|
99
|
+
yield(self.class.new(full_path))
|
100
|
+
end
|
101
|
+
end
|
102
|
+
|
103
|
+
# Moves this file to the `target_path` by copying it to the new name and then deleting the current file.
|
104
|
+
#
|
105
|
+
# Notes:
|
106
|
+
# - Can copy across buckets.
|
107
|
+
def move_to(target_path)
|
108
|
+
target = IOStreams.new(target_path)
|
109
|
+
return super(target) unless target.is_a?(self.class)
|
110
|
+
|
111
|
+
target.mkpath
|
112
|
+
# In case the file is being moved across partitions
|
113
|
+
FileUtils.move(path, target.to_s)
|
114
|
+
target
|
115
|
+
end
|
116
|
+
|
117
|
+
def mkpath
|
118
|
+
dir = ::File.dirname(path)
|
119
|
+
FileUtils.mkdir_p(dir) unless ::File.exist?(dir)
|
120
|
+
self
|
121
|
+
end
|
122
|
+
|
123
|
+
def mkdir
|
124
|
+
FileUtils.mkdir_p(path) unless ::File.exist?(path)
|
125
|
+
self
|
126
|
+
end
|
127
|
+
|
128
|
+
def exist?
|
129
|
+
::File.exist?(path)
|
130
|
+
end
|
131
|
+
|
132
|
+
def size
|
133
|
+
::File.size(path)
|
134
|
+
end
|
135
|
+
|
136
|
+
def delete
|
137
|
+
return self unless exist?
|
138
|
+
|
139
|
+
::File.directory?(path) ? Dir.delete(path) : ::File.unlink(path)
|
140
|
+
self
|
141
|
+
end
|
142
|
+
|
143
|
+
def delete_all
|
144
|
+
return self unless exist?
|
145
|
+
|
146
|
+
::File.directory?(path) ? FileUtils.remove_dir(path) : ::File.unlink(path)
|
147
|
+
self
|
148
|
+
end
|
149
|
+
|
150
|
+
# Returns the real path by stripping `.`, `..` and expands any symlinks.
|
151
|
+
def realpath
|
152
|
+
self.class.new(::File.realpath(path))
|
153
|
+
end
|
154
|
+
|
155
|
+
# Read from file
|
156
|
+
def reader(&block)
|
157
|
+
::File.open(path, "rb") { |io| streams.reader(io, &block) }
|
158
|
+
end
|
159
|
+
|
160
|
+
# Write to file
|
161
|
+
#
|
162
|
+
# Note:
|
163
|
+
# If an exception is raised whilst the file is being written to the file is removed to
|
164
|
+
# prevent incomplete / partial files from being created.
|
165
|
+
def writer(create_path: true, &block)
|
166
|
+
mkpath if create_path
|
167
|
+
begin
|
168
|
+
::File.open(path, "wb") { |io| streams.writer(io, &block) }
|
169
|
+
rescue StandardError => e
|
170
|
+
::File.unlink(path) if ::File.exist?(path)
|
171
|
+
raise(e)
|
172
|
+
end
|
173
|
+
end
|
174
|
+
end
|
175
|
+
end
|
176
|
+
end
|
@@ -0,0 +1,92 @@
|
|
1
|
+
require 'net/http'
|
2
|
+
require 'uri'
|
3
|
+
module IOStreams
|
4
|
+
module Paths
|
5
|
+
class HTTP < IOStreams::Path
|
6
|
+
attr_reader :username, :password, :http_redirect_count
|
7
|
+
|
8
|
+
# Stream to/from a remote file over http(s).
|
9
|
+
#
|
10
|
+
# Parameters:
|
11
|
+
# url: [String]
|
12
|
+
# URI of the file to download.
|
13
|
+
# Example:
|
14
|
+
# https://www5.fdic.gov/idasp/Offices2.zip
|
15
|
+
# http://hostname/path/file_name
|
16
|
+
#
|
17
|
+
# Full url showing all the optional elements that can be set via the url:
|
18
|
+
# https://username:password@hostname/path/file_name
|
19
|
+
#
|
20
|
+
# username: [String]
|
21
|
+
# When supplied, basic authentication is used with the username and password.
|
22
|
+
#
|
23
|
+
# password: [String]
|
24
|
+
# Password to use use with basic authentication when the username is supplied.
|
25
|
+
#
|
26
|
+
# http_redirect_count: [Integer]
|
27
|
+
# Maximum number of http redirects to follow.
|
28
|
+
def initialize(url, username: nil, password: nil, http_redirect_count: 10)
|
29
|
+
uri = URI.parse(url)
|
30
|
+
unless %w[http https].include?(uri.scheme)
|
31
|
+
raise(ArgumentError, "Invalid URL. Required Format: 'http://<host_name>/<file_name>', or 'https://<host_name>/<file_name>'")
|
32
|
+
end
|
33
|
+
|
34
|
+
@username = username || uri.user
|
35
|
+
@password = password || uri.password
|
36
|
+
@http_redirect_count = http_redirect_count
|
37
|
+
super(url)
|
38
|
+
end
|
39
|
+
|
40
|
+
# Read a file using an http get.
|
41
|
+
#
|
42
|
+
# For example:
|
43
|
+
# IOStreams.path('https://www5.fdic.gov/idasp/Offices2.zip').reader {|file| puts file.read}
|
44
|
+
#
|
45
|
+
# Read the file without unzipping and streaming the first file in the zip:
|
46
|
+
# IOStreams.path('https://www5.fdic.gov/idasp/Offices2.zip').stream(:none).reader {|file| puts file.read}
|
47
|
+
#
|
48
|
+
# Notes:
|
49
|
+
# * Since Net::HTTP download only supports a push stream, the data is streamed into a tempfile first.
|
50
|
+
def reader(&block)
|
51
|
+
handle_redirects(path, http_redirect_count, &block)
|
52
|
+
end
|
53
|
+
|
54
|
+
def handle_redirects(uri, http_redirect_count, &block)
|
55
|
+
uri = URI.parse(uri) unless uri.is_a?(URI)
|
56
|
+
result = nil
|
57
|
+
raise(IOStreams::Errors::CommunicationsFailure, "Too many redirects") if http_redirect_count < 1
|
58
|
+
|
59
|
+
Net::HTTP.start(uri.hostname, uri.port, use_ssl: uri.scheme == 'https') do |http|
|
60
|
+
request = Net::HTTP::Get.new(uri)
|
61
|
+
request.basic_auth(username, password) if username
|
62
|
+
|
63
|
+
http.request(request) do |response|
|
64
|
+
if response.is_a?(Net::HTTPNotFound)
|
65
|
+
raise(IOStreams::Errors::CommunicationsFailure, "Invalid URL: #{uri}")
|
66
|
+
end
|
67
|
+
if response.is_a?(Net::HTTPUnauthorized)
|
68
|
+
raise(IOStreams::Errors::CommunicationsFailure, "Authorization Required: Invalid :username or :password.")
|
69
|
+
end
|
70
|
+
|
71
|
+
if response.is_a?(Net::HTTPRedirection)
|
72
|
+
new_uri = response['location']
|
73
|
+
return handle_redirects(new_uri, http_redirect_count: http_redirect_count - 1, &block)
|
74
|
+
end
|
75
|
+
|
76
|
+
unless response.is_a?(Net::HTTPSuccess)
|
77
|
+
raise(IOStreams::Errors::CommunicationsFailure, "Invalid response code: #{response.code}")
|
78
|
+
end
|
79
|
+
|
80
|
+
# Since Net::HTTP download only supports a push stream, write it to a tempfile first.
|
81
|
+
Utils.temp_file_name('iostreams_http') do |file_name|
|
82
|
+
::File.open(file_name, 'wb') { |io| response.read_body { |chunk| io.write(chunk) } }
|
83
|
+
# Return a read stream
|
84
|
+
result = ::File.open(file_name, 'rb', &block)
|
85
|
+
end
|
86
|
+
end
|
87
|
+
end
|
88
|
+
result
|
89
|
+
end
|
90
|
+
end
|
91
|
+
end
|
92
|
+
end
|