iostreams 0.20.3 → 1.0.0.beta
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/io_streams/bzip2/reader.rb +9 -21
- data/lib/io_streams/bzip2/writer.rb +9 -21
- data/lib/io_streams/deprecated.rb +217 -0
- data/lib/io_streams/encode/reader.rb +12 -16
- data/lib/io_streams/encode/writer.rb +9 -13
- data/lib/io_streams/errors.rb +6 -6
- data/lib/io_streams/gzip/reader.rb +7 -14
- data/lib/io_streams/gzip/writer.rb +7 -15
- data/lib/io_streams/io_streams.rb +182 -524
- data/lib/io_streams/line/reader.rb +9 -9
- data/lib/io_streams/line/writer.rb +10 -11
- data/lib/io_streams/path.rb +190 -0
- data/lib/io_streams/paths/file.rb +176 -0
- data/lib/io_streams/paths/http.rb +92 -0
- data/lib/io_streams/paths/matcher.rb +61 -0
- data/lib/io_streams/paths/s3.rb +269 -0
- data/lib/io_streams/paths/sftp.rb +99 -0
- data/lib/io_streams/pgp.rb +47 -19
- data/lib/io_streams/pgp/reader.rb +20 -28
- data/lib/io_streams/pgp/writer.rb +24 -46
- data/lib/io_streams/reader.rb +28 -0
- data/lib/io_streams/record/reader.rb +20 -16
- data/lib/io_streams/record/writer.rb +28 -28
- data/lib/io_streams/row/reader.rb +22 -26
- data/lib/io_streams/row/writer.rb +29 -28
- data/lib/io_streams/stream.rb +400 -0
- data/lib/io_streams/streams.rb +125 -0
- data/lib/io_streams/symmetric_encryption/reader.rb +5 -13
- data/lib/io_streams/symmetric_encryption/writer.rb +16 -15
- data/lib/io_streams/tabular/header.rb +9 -3
- data/lib/io_streams/tabular/parser/array.rb +8 -3
- data/lib/io_streams/tabular/parser/csv.rb +6 -2
- data/lib/io_streams/tabular/parser/hash.rb +4 -1
- data/lib/io_streams/tabular/parser/json.rb +3 -1
- data/lib/io_streams/tabular/parser/psv.rb +3 -1
- data/lib/io_streams/tabular/utility/csv_row.rb +9 -8
- data/lib/io_streams/utils.rb +22 -0
- data/lib/io_streams/version.rb +1 -1
- data/lib/io_streams/writer.rb +28 -0
- data/lib/io_streams/xlsx/reader.rb +7 -19
- data/lib/io_streams/zip/reader.rb +7 -26
- data/lib/io_streams/zip/writer.rb +21 -38
- data/lib/iostreams.rb +15 -15
- data/test/bzip2_reader_test.rb +3 -3
- data/test/bzip2_writer_test.rb +3 -3
- data/test/deprecated_test.rb +123 -0
- data/test/encode_reader_test.rb +3 -3
- data/test/encode_writer_test.rb +6 -6
- data/test/gzip_reader_test.rb +2 -2
- data/test/gzip_writer_test.rb +3 -3
- data/test/io_streams_test.rb +43 -136
- data/test/line_reader_test.rb +20 -20
- data/test/line_writer_test.rb +3 -3
- data/test/path_test.rb +30 -28
- data/test/paths/file_test.rb +206 -0
- data/test/paths/http_test.rb +34 -0
- data/test/paths/matcher_test.rb +111 -0
- data/test/paths/s3_test.rb +207 -0
- data/test/pgp_reader_test.rb +8 -8
- data/test/pgp_writer_test.rb +13 -13
- data/test/record_reader_test.rb +5 -5
- data/test/record_writer_test.rb +4 -4
- data/test/row_reader_test.rb +5 -5
- data/test/row_writer_test.rb +6 -6
- data/test/stream_test.rb +116 -0
- data/test/streams_test.rb +255 -0
- data/test/utils_test.rb +20 -0
- data/test/xlsx_reader_test.rb +3 -3
- data/test/zip_reader_test.rb +12 -12
- data/test/zip_writer_test.rb +5 -5
- metadata +33 -45
- data/lib/io_streams/base_path.rb +0 -72
- data/lib/io_streams/file/path.rb +0 -58
- data/lib/io_streams/file/reader.rb +0 -12
- data/lib/io_streams/file/writer.rb +0 -22
- data/lib/io_streams/http/reader.rb +0 -71
- data/lib/io_streams/s3.rb +0 -26
- data/lib/io_streams/s3/path.rb +0 -40
- data/lib/io_streams/s3/reader.rb +0 -28
- data/lib/io_streams/s3/writer.rb +0 -85
- data/lib/io_streams/sftp/reader.rb +0 -67
- data/lib/io_streams/sftp/writer.rb +0 -68
- data/test/base_path_test.rb +0 -35
- data/test/file_path_test.rb +0 -97
- data/test/file_reader_test.rb +0 -33
- data/test/file_writer_test.rb +0 -50
- data/test/http_reader_test.rb +0 -38
- data/test/s3_reader_test.rb +0 -41
- data/test/s3_writer_test.rb +0 -41
data/test/file_path_test.rb
DELETED
@@ -1,97 +0,0 @@
|
|
1
|
-
require_relative 'test_helper'
|
2
|
-
|
3
|
-
module IOStreams
|
4
|
-
class FilePathTest < Minitest::Test
|
5
|
-
describe IOStreams::File::Path do
|
6
|
-
let(:path) { IOStreams::File::Path.new('/tmp/iostreams/some_test_path').delete(recursively: true) }
|
7
|
-
let(:path_with_file_name) { IOStreams::File::Path.new('/tmp/iostreams/some_test_path/test_file.txt').delete }
|
8
|
-
|
9
|
-
describe '.temp_file_name' do
|
10
|
-
it 'returns value from block' do
|
11
|
-
result = IOStreams::File::Path.temp_file_name('base', '.ext') { |name| 257 }
|
12
|
-
assert_equal 257, result
|
13
|
-
end
|
14
|
-
|
15
|
-
it 'supplies new temp file_name' do
|
16
|
-
file_name = nil
|
17
|
-
file_name2 = nil
|
18
|
-
IOStreams::File::Path.temp_file_name('base', '.ext') { |name| file_name = name }
|
19
|
-
IOStreams::File::Path.temp_file_name('base', '.ext') { |name| file_name2 = name }
|
20
|
-
refute_equal file_name, file_name2
|
21
|
-
end
|
22
|
-
end
|
23
|
-
|
24
|
-
describe '.mkpath' do
|
25
|
-
it 'makes path skipping file_name' do
|
26
|
-
new_path = path.join('test_mkpath.xls')
|
27
|
-
IOStreams::File::Path.mkpath(new_path.to_s)
|
28
|
-
|
29
|
-
assert ::File.exist?(path.to_s)
|
30
|
-
refute ::File.exist?(new_path.to_s)
|
31
|
-
end
|
32
|
-
end
|
33
|
-
|
34
|
-
describe '#mkpath' do
|
35
|
-
it 'makes path skipping file_name' do
|
36
|
-
#path.writer { |io| io << "Hello World" }
|
37
|
-
new_path = path.join('test_mkpath.xls').mkpath
|
38
|
-
assert ::File.exist?(path.to_s)
|
39
|
-
refute ::File.exist?(new_path.to_s)
|
40
|
-
end
|
41
|
-
end
|
42
|
-
|
43
|
-
describe '#mkdir' do
|
44
|
-
it 'makes entire path that does not have a file name' do
|
45
|
-
new_path = path.join('more_path').mkdir
|
46
|
-
assert ::File.exist?(path.to_s)
|
47
|
-
assert ::File.exist?(new_path.to_s)
|
48
|
-
end
|
49
|
-
end
|
50
|
-
|
51
|
-
describe '#exist?' do
|
52
|
-
it 'true on existing file' do
|
53
|
-
new_path = path.join('test_exist.txt')
|
54
|
-
new_path.writer { |io| io << "Hello World" }
|
55
|
-
assert ::File.exist?(path.to_s)
|
56
|
-
assert ::File.exist?(new_path.to_s)
|
57
|
-
end
|
58
|
-
end
|
59
|
-
|
60
|
-
describe '#size' do
|
61
|
-
it 'of file' do
|
62
|
-
data = "Hello World"
|
63
|
-
path_with_file_name.writer { |io| io << data }
|
64
|
-
assert_equal data.size, path_with_file_name.size
|
65
|
-
end
|
66
|
-
end
|
67
|
-
|
68
|
-
describe '#delete' do
|
69
|
-
it 'deletes existing file' do
|
70
|
-
path_with_file_name.writer { |io| io << "Hello World" }
|
71
|
-
assert ::File.exist?(path_with_file_name.to_s)
|
72
|
-
path_with_file_name.delete
|
73
|
-
refute ::File.exist?(path_with_file_name.to_s)
|
74
|
-
end
|
75
|
-
|
76
|
-
it 'ignores missing file' do
|
77
|
-
path_with_file_name.delete
|
78
|
-
path_with_file_name.delete
|
79
|
-
end
|
80
|
-
end
|
81
|
-
|
82
|
-
describe 'reader' do
|
83
|
-
it 'reads file' do
|
84
|
-
path_with_file_name.writer { |io| io << "Hello World" }
|
85
|
-
assert_equal "Hello World", path_with_file_name.reader(&:read)
|
86
|
-
end
|
87
|
-
end
|
88
|
-
|
89
|
-
describe 'writer' do
|
90
|
-
it 'creates file' do
|
91
|
-
path_with_file_name.writer { |io| io << "Hello World" }
|
92
|
-
assert ::File.exist?(path_with_file_name.to_s)
|
93
|
-
end
|
94
|
-
end
|
95
|
-
end
|
96
|
-
end
|
97
|
-
end
|
data/test/file_reader_test.rb
DELETED
@@ -1,33 +0,0 @@
|
|
1
|
-
require_relative 'test_helper'
|
2
|
-
|
3
|
-
class FileReaderTest < Minitest::Test
|
4
|
-
describe IOStreams::File::Reader do
|
5
|
-
let :file_name do
|
6
|
-
File.join(File.dirname(__FILE__), 'files', 'text.txt')
|
7
|
-
end
|
8
|
-
|
9
|
-
let :raw do
|
10
|
-
File.read(file_name)
|
11
|
-
end
|
12
|
-
|
13
|
-
describe '.open' do
|
14
|
-
it 'file' do
|
15
|
-
result = IOStreams::File::Reader.open(file_name) do |io|
|
16
|
-
io.read
|
17
|
-
end
|
18
|
-
assert_equal raw, result
|
19
|
-
end
|
20
|
-
|
21
|
-
it 'does not support streams' do
|
22
|
-
assert_raises ArgumentError do
|
23
|
-
File.open(file_name) do |file|
|
24
|
-
IOStreams::File::Reader.open(file) do |io|
|
25
|
-
io.read
|
26
|
-
end
|
27
|
-
end
|
28
|
-
end
|
29
|
-
end
|
30
|
-
end
|
31
|
-
|
32
|
-
end
|
33
|
-
end
|
data/test/file_writer_test.rb
DELETED
@@ -1,50 +0,0 @@
|
|
1
|
-
require_relative 'test_helper'
|
2
|
-
|
3
|
-
class FileWriterTest < Minitest::Test
|
4
|
-
describe IOStreams::File::Writer do
|
5
|
-
let :file_name do
|
6
|
-
File.join(File.dirname(__FILE__), 'files', 'text.txt')
|
7
|
-
end
|
8
|
-
|
9
|
-
let :raw do
|
10
|
-
File.read(file_name)
|
11
|
-
end
|
12
|
-
|
13
|
-
describe '.open' do
|
14
|
-
it 'file' do
|
15
|
-
temp_file = Tempfile.new('rocket_job')
|
16
|
-
file_name = temp_file.to_path
|
17
|
-
IOStreams::File::Writer.open(file_name) do |io|
|
18
|
-
io.write(raw)
|
19
|
-
end
|
20
|
-
result = File.read(file_name)
|
21
|
-
assert_equal raw, result
|
22
|
-
end
|
23
|
-
|
24
|
-
it 'cleans up failed upload' do
|
25
|
-
temp_file = Tempfile.new('rocket_job')
|
26
|
-
file_name = temp_file.to_path
|
27
|
-
begin
|
28
|
-
IOStreams::File::Writer.open(file_name) do |io|
|
29
|
-
io.write("hello world")
|
30
|
-
assert File.exist?(file_name)
|
31
|
-
raise(ArgumentError, "Oh no")
|
32
|
-
end
|
33
|
-
rescue ArgumentError
|
34
|
-
end
|
35
|
-
|
36
|
-
refute File.exist?(file_name)
|
37
|
-
end
|
38
|
-
|
39
|
-
it 'does not support streams' do
|
40
|
-
io_string = StringIO.new
|
41
|
-
assert_raises ArgumentError do
|
42
|
-
IOStreams::File::Writer.open(io_string) do |io|
|
43
|
-
io.write(raw)
|
44
|
-
end
|
45
|
-
end
|
46
|
-
end
|
47
|
-
end
|
48
|
-
|
49
|
-
end
|
50
|
-
end
|
data/test/http_reader_test.rb
DELETED
@@ -1,38 +0,0 @@
|
|
1
|
-
require_relative 'test_helper'
|
2
|
-
|
3
|
-
class HTTPReaderTest < Minitest::Test
|
4
|
-
describe IOStreams::HTTP::Reader do
|
5
|
-
let :uri do
|
6
|
-
"http://example.com/index.html?count=10"
|
7
|
-
end
|
8
|
-
|
9
|
-
let :ssl_uri do
|
10
|
-
"https://example.com/index.html?count=10"
|
11
|
-
end
|
12
|
-
|
13
|
-
describe '.open' do
|
14
|
-
it 'reads http' do
|
15
|
-
result = IOStreams::HTTP::Reader.open(uri) do |io|
|
16
|
-
io.read
|
17
|
-
end
|
18
|
-
assert_includes result, "<html>"
|
19
|
-
end
|
20
|
-
|
21
|
-
it 'reads https' do
|
22
|
-
result = IOStreams::HTTP::Reader.open(ssl_uri) do |io|
|
23
|
-
io.read
|
24
|
-
end
|
25
|
-
assert_includes result, "<html>"
|
26
|
-
end
|
27
|
-
|
28
|
-
it 'does not support streams' do
|
29
|
-
assert_raises ArgumentError do
|
30
|
-
io = StringIO.new
|
31
|
-
IOStreams::HTTP::Reader.open(io) do |http_io|
|
32
|
-
http_io.read
|
33
|
-
end
|
34
|
-
end
|
35
|
-
end
|
36
|
-
end
|
37
|
-
end
|
38
|
-
end
|
data/test/s3_reader_test.rb
DELETED
@@ -1,41 +0,0 @@
|
|
1
|
-
require_relative 'test_helper'
|
2
|
-
|
3
|
-
class S3ReaderTest < Minitest::Test
|
4
|
-
describe IOStreams::File::Reader do
|
5
|
-
let :file_name do
|
6
|
-
File.join(File.dirname(__FILE__), 'files', 'text.txt')
|
7
|
-
end
|
8
|
-
|
9
|
-
let :raw do
|
10
|
-
File.read(file_name)
|
11
|
-
end
|
12
|
-
|
13
|
-
let :uri do
|
14
|
-
"s3://#{ENV['S3_BUCKET_NAME']}/s3_test/test.txt"
|
15
|
-
end
|
16
|
-
|
17
|
-
let :upload_s3_file do
|
18
|
-
IOStreams::S3::Writer.open(uri) { |io| io << raw }
|
19
|
-
end
|
20
|
-
|
21
|
-
describe '.open' do
|
22
|
-
it 'reads' do
|
23
|
-
unless ENV['S3_BUCKET_NAME']
|
24
|
-
skip "Supply 'S3_BUCKET_NAME' environment variable with S3 bucket name to test with"
|
25
|
-
end
|
26
|
-
|
27
|
-
upload_s3_file
|
28
|
-
result = IOStreams::S3::Reader.open(uri) { |io| io.read }
|
29
|
-
assert_equal raw, result
|
30
|
-
end
|
31
|
-
|
32
|
-
it 'does not support streams' do
|
33
|
-
io_string = StringIO.new('data')
|
34
|
-
assert_raises ArgumentError do
|
35
|
-
IOStreams::S3::Reader.open(io_string) { |io| io.read }
|
36
|
-
end
|
37
|
-
end
|
38
|
-
end
|
39
|
-
|
40
|
-
end
|
41
|
-
end
|
data/test/s3_writer_test.rb
DELETED
@@ -1,41 +0,0 @@
|
|
1
|
-
require_relative 'test_helper'
|
2
|
-
|
3
|
-
class FileWriterTest < Minitest::Test
|
4
|
-
describe IOStreams::File::Writer do
|
5
|
-
let :file_name do
|
6
|
-
File.join(File.dirname(__FILE__), 'files', 'text.txt')
|
7
|
-
end
|
8
|
-
|
9
|
-
let :raw do
|
10
|
-
File.read(file_name)
|
11
|
-
end
|
12
|
-
|
13
|
-
let :uri do
|
14
|
-
"s3://#{ENV['S3_BUCKET_NAME']}/s3_test/test.txt"
|
15
|
-
end
|
16
|
-
|
17
|
-
let :upload_s3_file do
|
18
|
-
IOStreams::S3::Writer.open(uri) { |io| io << raw }
|
19
|
-
end
|
20
|
-
|
21
|
-
describe '.open' do
|
22
|
-
it 'writes' do
|
23
|
-
unless ENV['S3_BUCKET_NAME']
|
24
|
-
skip "Supply 'S3_BUCKET_NAME' environment variable with S3 bucket name to test with"
|
25
|
-
end
|
26
|
-
|
27
|
-
IOStreams::S3::Writer.open(uri) { |io| io.write(raw) }
|
28
|
-
result = IOStreams::S3::Reader.open(uri) { |io| io.read }
|
29
|
-
assert_equal raw, result
|
30
|
-
end
|
31
|
-
|
32
|
-
it 'does not support streams' do
|
33
|
-
io_string = StringIO.new
|
34
|
-
assert_raises ArgumentError do
|
35
|
-
IOStreams::S3::Writer.open(io_string) { |io| io.write(raw) }
|
36
|
-
end
|
37
|
-
end
|
38
|
-
end
|
39
|
-
|
40
|
-
end
|
41
|
-
end
|