iostreams 0.20.3 → 1.0.0.beta
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/io_streams/bzip2/reader.rb +9 -21
- data/lib/io_streams/bzip2/writer.rb +9 -21
- data/lib/io_streams/deprecated.rb +217 -0
- data/lib/io_streams/encode/reader.rb +12 -16
- data/lib/io_streams/encode/writer.rb +9 -13
- data/lib/io_streams/errors.rb +6 -6
- data/lib/io_streams/gzip/reader.rb +7 -14
- data/lib/io_streams/gzip/writer.rb +7 -15
- data/lib/io_streams/io_streams.rb +182 -524
- data/lib/io_streams/line/reader.rb +9 -9
- data/lib/io_streams/line/writer.rb +10 -11
- data/lib/io_streams/path.rb +190 -0
- data/lib/io_streams/paths/file.rb +176 -0
- data/lib/io_streams/paths/http.rb +92 -0
- data/lib/io_streams/paths/matcher.rb +61 -0
- data/lib/io_streams/paths/s3.rb +269 -0
- data/lib/io_streams/paths/sftp.rb +99 -0
- data/lib/io_streams/pgp.rb +47 -19
- data/lib/io_streams/pgp/reader.rb +20 -28
- data/lib/io_streams/pgp/writer.rb +24 -46
- data/lib/io_streams/reader.rb +28 -0
- data/lib/io_streams/record/reader.rb +20 -16
- data/lib/io_streams/record/writer.rb +28 -28
- data/lib/io_streams/row/reader.rb +22 -26
- data/lib/io_streams/row/writer.rb +29 -28
- data/lib/io_streams/stream.rb +400 -0
- data/lib/io_streams/streams.rb +125 -0
- data/lib/io_streams/symmetric_encryption/reader.rb +5 -13
- data/lib/io_streams/symmetric_encryption/writer.rb +16 -15
- data/lib/io_streams/tabular/header.rb +9 -3
- data/lib/io_streams/tabular/parser/array.rb +8 -3
- data/lib/io_streams/tabular/parser/csv.rb +6 -2
- data/lib/io_streams/tabular/parser/hash.rb +4 -1
- data/lib/io_streams/tabular/parser/json.rb +3 -1
- data/lib/io_streams/tabular/parser/psv.rb +3 -1
- data/lib/io_streams/tabular/utility/csv_row.rb +9 -8
- data/lib/io_streams/utils.rb +22 -0
- data/lib/io_streams/version.rb +1 -1
- data/lib/io_streams/writer.rb +28 -0
- data/lib/io_streams/xlsx/reader.rb +7 -19
- data/lib/io_streams/zip/reader.rb +7 -26
- data/lib/io_streams/zip/writer.rb +21 -38
- data/lib/iostreams.rb +15 -15
- data/test/bzip2_reader_test.rb +3 -3
- data/test/bzip2_writer_test.rb +3 -3
- data/test/deprecated_test.rb +123 -0
- data/test/encode_reader_test.rb +3 -3
- data/test/encode_writer_test.rb +6 -6
- data/test/gzip_reader_test.rb +2 -2
- data/test/gzip_writer_test.rb +3 -3
- data/test/io_streams_test.rb +43 -136
- data/test/line_reader_test.rb +20 -20
- data/test/line_writer_test.rb +3 -3
- data/test/path_test.rb +30 -28
- data/test/paths/file_test.rb +206 -0
- data/test/paths/http_test.rb +34 -0
- data/test/paths/matcher_test.rb +111 -0
- data/test/paths/s3_test.rb +207 -0
- data/test/pgp_reader_test.rb +8 -8
- data/test/pgp_writer_test.rb +13 -13
- data/test/record_reader_test.rb +5 -5
- data/test/record_writer_test.rb +4 -4
- data/test/row_reader_test.rb +5 -5
- data/test/row_writer_test.rb +6 -6
- data/test/stream_test.rb +116 -0
- data/test/streams_test.rb +255 -0
- data/test/utils_test.rb +20 -0
- data/test/xlsx_reader_test.rb +3 -3
- data/test/zip_reader_test.rb +12 -12
- data/test/zip_writer_test.rb +5 -5
- metadata +33 -45
- data/lib/io_streams/base_path.rb +0 -72
- data/lib/io_streams/file/path.rb +0 -58
- data/lib/io_streams/file/reader.rb +0 -12
- data/lib/io_streams/file/writer.rb +0 -22
- data/lib/io_streams/http/reader.rb +0 -71
- data/lib/io_streams/s3.rb +0 -26
- data/lib/io_streams/s3/path.rb +0 -40
- data/lib/io_streams/s3/reader.rb +0 -28
- data/lib/io_streams/s3/writer.rb +0 -85
- data/lib/io_streams/sftp/reader.rb +0 -67
- data/lib/io_streams/sftp/writer.rb +0 -68
- data/test/base_path_test.rb +0 -35
- data/test/file_path_test.rb +0 -97
- data/test/file_reader_test.rb +0 -33
- data/test/file_writer_test.rb +0 -50
- data/test/http_reader_test.rb +0 -38
- data/test/s3_reader_test.rb +0 -41
- data/test/s3_writer_test.rb +0 -41
data/test/file_path_test.rb
DELETED
@@ -1,97 +0,0 @@
|
|
1
|
-
require_relative 'test_helper'
|
2
|
-
|
3
|
-
module IOStreams
|
4
|
-
class FilePathTest < Minitest::Test
|
5
|
-
describe IOStreams::File::Path do
|
6
|
-
let(:path) { IOStreams::File::Path.new('/tmp/iostreams/some_test_path').delete(recursively: true) }
|
7
|
-
let(:path_with_file_name) { IOStreams::File::Path.new('/tmp/iostreams/some_test_path/test_file.txt').delete }
|
8
|
-
|
9
|
-
describe '.temp_file_name' do
|
10
|
-
it 'returns value from block' do
|
11
|
-
result = IOStreams::File::Path.temp_file_name('base', '.ext') { |name| 257 }
|
12
|
-
assert_equal 257, result
|
13
|
-
end
|
14
|
-
|
15
|
-
it 'supplies new temp file_name' do
|
16
|
-
file_name = nil
|
17
|
-
file_name2 = nil
|
18
|
-
IOStreams::File::Path.temp_file_name('base', '.ext') { |name| file_name = name }
|
19
|
-
IOStreams::File::Path.temp_file_name('base', '.ext') { |name| file_name2 = name }
|
20
|
-
refute_equal file_name, file_name2
|
21
|
-
end
|
22
|
-
end
|
23
|
-
|
24
|
-
describe '.mkpath' do
|
25
|
-
it 'makes path skipping file_name' do
|
26
|
-
new_path = path.join('test_mkpath.xls')
|
27
|
-
IOStreams::File::Path.mkpath(new_path.to_s)
|
28
|
-
|
29
|
-
assert ::File.exist?(path.to_s)
|
30
|
-
refute ::File.exist?(new_path.to_s)
|
31
|
-
end
|
32
|
-
end
|
33
|
-
|
34
|
-
describe '#mkpath' do
|
35
|
-
it 'makes path skipping file_name' do
|
36
|
-
#path.writer { |io| io << "Hello World" }
|
37
|
-
new_path = path.join('test_mkpath.xls').mkpath
|
38
|
-
assert ::File.exist?(path.to_s)
|
39
|
-
refute ::File.exist?(new_path.to_s)
|
40
|
-
end
|
41
|
-
end
|
42
|
-
|
43
|
-
describe '#mkdir' do
|
44
|
-
it 'makes entire path that does not have a file name' do
|
45
|
-
new_path = path.join('more_path').mkdir
|
46
|
-
assert ::File.exist?(path.to_s)
|
47
|
-
assert ::File.exist?(new_path.to_s)
|
48
|
-
end
|
49
|
-
end
|
50
|
-
|
51
|
-
describe '#exist?' do
|
52
|
-
it 'true on existing file' do
|
53
|
-
new_path = path.join('test_exist.txt')
|
54
|
-
new_path.writer { |io| io << "Hello World" }
|
55
|
-
assert ::File.exist?(path.to_s)
|
56
|
-
assert ::File.exist?(new_path.to_s)
|
57
|
-
end
|
58
|
-
end
|
59
|
-
|
60
|
-
describe '#size' do
|
61
|
-
it 'of file' do
|
62
|
-
data = "Hello World"
|
63
|
-
path_with_file_name.writer { |io| io << data }
|
64
|
-
assert_equal data.size, path_with_file_name.size
|
65
|
-
end
|
66
|
-
end
|
67
|
-
|
68
|
-
describe '#delete' do
|
69
|
-
it 'deletes existing file' do
|
70
|
-
path_with_file_name.writer { |io| io << "Hello World" }
|
71
|
-
assert ::File.exist?(path_with_file_name.to_s)
|
72
|
-
path_with_file_name.delete
|
73
|
-
refute ::File.exist?(path_with_file_name.to_s)
|
74
|
-
end
|
75
|
-
|
76
|
-
it 'ignores missing file' do
|
77
|
-
path_with_file_name.delete
|
78
|
-
path_with_file_name.delete
|
79
|
-
end
|
80
|
-
end
|
81
|
-
|
82
|
-
describe 'reader' do
|
83
|
-
it 'reads file' do
|
84
|
-
path_with_file_name.writer { |io| io << "Hello World" }
|
85
|
-
assert_equal "Hello World", path_with_file_name.reader(&:read)
|
86
|
-
end
|
87
|
-
end
|
88
|
-
|
89
|
-
describe 'writer' do
|
90
|
-
it 'creates file' do
|
91
|
-
path_with_file_name.writer { |io| io << "Hello World" }
|
92
|
-
assert ::File.exist?(path_with_file_name.to_s)
|
93
|
-
end
|
94
|
-
end
|
95
|
-
end
|
96
|
-
end
|
97
|
-
end
|
data/test/file_reader_test.rb
DELETED
@@ -1,33 +0,0 @@
|
|
1
|
-
require_relative 'test_helper'
|
2
|
-
|
3
|
-
class FileReaderTest < Minitest::Test
|
4
|
-
describe IOStreams::File::Reader do
|
5
|
-
let :file_name do
|
6
|
-
File.join(File.dirname(__FILE__), 'files', 'text.txt')
|
7
|
-
end
|
8
|
-
|
9
|
-
let :raw do
|
10
|
-
File.read(file_name)
|
11
|
-
end
|
12
|
-
|
13
|
-
describe '.open' do
|
14
|
-
it 'file' do
|
15
|
-
result = IOStreams::File::Reader.open(file_name) do |io|
|
16
|
-
io.read
|
17
|
-
end
|
18
|
-
assert_equal raw, result
|
19
|
-
end
|
20
|
-
|
21
|
-
it 'does not support streams' do
|
22
|
-
assert_raises ArgumentError do
|
23
|
-
File.open(file_name) do |file|
|
24
|
-
IOStreams::File::Reader.open(file) do |io|
|
25
|
-
io.read
|
26
|
-
end
|
27
|
-
end
|
28
|
-
end
|
29
|
-
end
|
30
|
-
end
|
31
|
-
|
32
|
-
end
|
33
|
-
end
|
data/test/file_writer_test.rb
DELETED
@@ -1,50 +0,0 @@
|
|
1
|
-
require_relative 'test_helper'
|
2
|
-
|
3
|
-
class FileWriterTest < Minitest::Test
|
4
|
-
describe IOStreams::File::Writer do
|
5
|
-
let :file_name do
|
6
|
-
File.join(File.dirname(__FILE__), 'files', 'text.txt')
|
7
|
-
end
|
8
|
-
|
9
|
-
let :raw do
|
10
|
-
File.read(file_name)
|
11
|
-
end
|
12
|
-
|
13
|
-
describe '.open' do
|
14
|
-
it 'file' do
|
15
|
-
temp_file = Tempfile.new('rocket_job')
|
16
|
-
file_name = temp_file.to_path
|
17
|
-
IOStreams::File::Writer.open(file_name) do |io|
|
18
|
-
io.write(raw)
|
19
|
-
end
|
20
|
-
result = File.read(file_name)
|
21
|
-
assert_equal raw, result
|
22
|
-
end
|
23
|
-
|
24
|
-
it 'cleans up failed upload' do
|
25
|
-
temp_file = Tempfile.new('rocket_job')
|
26
|
-
file_name = temp_file.to_path
|
27
|
-
begin
|
28
|
-
IOStreams::File::Writer.open(file_name) do |io|
|
29
|
-
io.write("hello world")
|
30
|
-
assert File.exist?(file_name)
|
31
|
-
raise(ArgumentError, "Oh no")
|
32
|
-
end
|
33
|
-
rescue ArgumentError
|
34
|
-
end
|
35
|
-
|
36
|
-
refute File.exist?(file_name)
|
37
|
-
end
|
38
|
-
|
39
|
-
it 'does not support streams' do
|
40
|
-
io_string = StringIO.new
|
41
|
-
assert_raises ArgumentError do
|
42
|
-
IOStreams::File::Writer.open(io_string) do |io|
|
43
|
-
io.write(raw)
|
44
|
-
end
|
45
|
-
end
|
46
|
-
end
|
47
|
-
end
|
48
|
-
|
49
|
-
end
|
50
|
-
end
|
data/test/http_reader_test.rb
DELETED
@@ -1,38 +0,0 @@
|
|
1
|
-
require_relative 'test_helper'
|
2
|
-
|
3
|
-
class HTTPReaderTest < Minitest::Test
|
4
|
-
describe IOStreams::HTTP::Reader do
|
5
|
-
let :uri do
|
6
|
-
"http://example.com/index.html?count=10"
|
7
|
-
end
|
8
|
-
|
9
|
-
let :ssl_uri do
|
10
|
-
"https://example.com/index.html?count=10"
|
11
|
-
end
|
12
|
-
|
13
|
-
describe '.open' do
|
14
|
-
it 'reads http' do
|
15
|
-
result = IOStreams::HTTP::Reader.open(uri) do |io|
|
16
|
-
io.read
|
17
|
-
end
|
18
|
-
assert_includes result, "<html>"
|
19
|
-
end
|
20
|
-
|
21
|
-
it 'reads https' do
|
22
|
-
result = IOStreams::HTTP::Reader.open(ssl_uri) do |io|
|
23
|
-
io.read
|
24
|
-
end
|
25
|
-
assert_includes result, "<html>"
|
26
|
-
end
|
27
|
-
|
28
|
-
it 'does not support streams' do
|
29
|
-
assert_raises ArgumentError do
|
30
|
-
io = StringIO.new
|
31
|
-
IOStreams::HTTP::Reader.open(io) do |http_io|
|
32
|
-
http_io.read
|
33
|
-
end
|
34
|
-
end
|
35
|
-
end
|
36
|
-
end
|
37
|
-
end
|
38
|
-
end
|
data/test/s3_reader_test.rb
DELETED
@@ -1,41 +0,0 @@
|
|
1
|
-
require_relative 'test_helper'
|
2
|
-
|
3
|
-
class S3ReaderTest < Minitest::Test
|
4
|
-
describe IOStreams::File::Reader do
|
5
|
-
let :file_name do
|
6
|
-
File.join(File.dirname(__FILE__), 'files', 'text.txt')
|
7
|
-
end
|
8
|
-
|
9
|
-
let :raw do
|
10
|
-
File.read(file_name)
|
11
|
-
end
|
12
|
-
|
13
|
-
let :uri do
|
14
|
-
"s3://#{ENV['S3_BUCKET_NAME']}/s3_test/test.txt"
|
15
|
-
end
|
16
|
-
|
17
|
-
let :upload_s3_file do
|
18
|
-
IOStreams::S3::Writer.open(uri) { |io| io << raw }
|
19
|
-
end
|
20
|
-
|
21
|
-
describe '.open' do
|
22
|
-
it 'reads' do
|
23
|
-
unless ENV['S3_BUCKET_NAME']
|
24
|
-
skip "Supply 'S3_BUCKET_NAME' environment variable with S3 bucket name to test with"
|
25
|
-
end
|
26
|
-
|
27
|
-
upload_s3_file
|
28
|
-
result = IOStreams::S3::Reader.open(uri) { |io| io.read }
|
29
|
-
assert_equal raw, result
|
30
|
-
end
|
31
|
-
|
32
|
-
it 'does not support streams' do
|
33
|
-
io_string = StringIO.new('data')
|
34
|
-
assert_raises ArgumentError do
|
35
|
-
IOStreams::S3::Reader.open(io_string) { |io| io.read }
|
36
|
-
end
|
37
|
-
end
|
38
|
-
end
|
39
|
-
|
40
|
-
end
|
41
|
-
end
|
data/test/s3_writer_test.rb
DELETED
@@ -1,41 +0,0 @@
|
|
1
|
-
require_relative 'test_helper'
|
2
|
-
|
3
|
-
class FileWriterTest < Minitest::Test
|
4
|
-
describe IOStreams::File::Writer do
|
5
|
-
let :file_name do
|
6
|
-
File.join(File.dirname(__FILE__), 'files', 'text.txt')
|
7
|
-
end
|
8
|
-
|
9
|
-
let :raw do
|
10
|
-
File.read(file_name)
|
11
|
-
end
|
12
|
-
|
13
|
-
let :uri do
|
14
|
-
"s3://#{ENV['S3_BUCKET_NAME']}/s3_test/test.txt"
|
15
|
-
end
|
16
|
-
|
17
|
-
let :upload_s3_file do
|
18
|
-
IOStreams::S3::Writer.open(uri) { |io| io << raw }
|
19
|
-
end
|
20
|
-
|
21
|
-
describe '.open' do
|
22
|
-
it 'writes' do
|
23
|
-
unless ENV['S3_BUCKET_NAME']
|
24
|
-
skip "Supply 'S3_BUCKET_NAME' environment variable with S3 bucket name to test with"
|
25
|
-
end
|
26
|
-
|
27
|
-
IOStreams::S3::Writer.open(uri) { |io| io.write(raw) }
|
28
|
-
result = IOStreams::S3::Reader.open(uri) { |io| io.read }
|
29
|
-
assert_equal raw, result
|
30
|
-
end
|
31
|
-
|
32
|
-
it 'does not support streams' do
|
33
|
-
io_string = StringIO.new
|
34
|
-
assert_raises ArgumentError do
|
35
|
-
IOStreams::S3::Writer.open(io_string) { |io| io.write(raw) }
|
36
|
-
end
|
37
|
-
end
|
38
|
-
end
|
39
|
-
|
40
|
-
end
|
41
|
-
end
|