iostreams 0.14.0 → 0.15.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/LICENSE +202 -0
- data/README.md +155 -47
- data/lib/io_streams/file/reader.rb +7 -8
- data/lib/io_streams/file/writer.rb +7 -8
- data/lib/io_streams/io_streams.rb +313 -129
- data/lib/io_streams/{delimited → line}/reader.rb +20 -30
- data/lib/io_streams/line/writer.rb +81 -0
- data/lib/io_streams/pgp.rb +4 -14
- data/lib/io_streams/record/reader.rb +55 -0
- data/lib/io_streams/record/writer.rb +63 -0
- data/lib/io_streams/row/reader.rb +60 -0
- data/lib/io_streams/row/writer.rb +62 -0
- data/lib/io_streams/s3.rb +25 -0
- data/lib/io_streams/s3/reader.rb +64 -0
- data/lib/io_streams/s3/writer.rb +13 -0
- data/lib/io_streams/streams.rb +1 -1
- data/lib/io_streams/tabular.rb +163 -0
- data/lib/io_streams/tabular/errors.rb +14 -0
- data/lib/io_streams/tabular/header.rb +146 -0
- data/lib/io_streams/tabular/parser/array.rb +26 -0
- data/lib/io_streams/tabular/parser/base.rb +12 -0
- data/lib/io_streams/tabular/parser/csv.rb +35 -0
- data/lib/io_streams/tabular/parser/fixed.rb +88 -0
- data/lib/io_streams/tabular/parser/hash.rb +21 -0
- data/lib/io_streams/tabular/parser/json.rb +25 -0
- data/lib/io_streams/tabular/parser/psv.rb +34 -0
- data/lib/io_streams/tabular/utility/csv_row.rb +115 -0
- data/lib/io_streams/version.rb +2 -2
- data/lib/io_streams/xlsx/reader.rb +1 -1
- data/lib/io_streams/zip/reader.rb +1 -1
- data/lib/io_streams/zip/writer.rb +1 -1
- data/lib/iostreams.rb +21 -10
- data/test/bzip2_reader_test.rb +21 -22
- data/test/bzip2_writer_test.rb +38 -32
- data/test/file_reader_test.rb +19 -18
- data/test/file_writer_test.rb +23 -22
- data/test/files/test.json +3 -0
- data/test/gzip_reader_test.rb +21 -22
- data/test/gzip_writer_test.rb +35 -29
- data/test/io_streams_test.rb +137 -61
- data/test/line_reader_test.rb +105 -0
- data/test/line_writer_test.rb +50 -0
- data/test/pgp_reader_test.rb +29 -29
- data/test/pgp_test.rb +149 -195
- data/test/pgp_writer_test.rb +63 -62
- data/test/record_reader_test.rb +61 -0
- data/test/record_writer_test.rb +73 -0
- data/test/row_reader_test.rb +34 -0
- data/test/row_writer_test.rb +51 -0
- data/test/tabular_test.rb +184 -0
- data/test/xlsx_reader_test.rb +13 -17
- data/test/zip_reader_test.rb +21 -22
- data/test/zip_writer_test.rb +40 -36
- metadata +41 -17
- data/lib/io_streams/csv/reader.rb +0 -21
- data/lib/io_streams/csv/writer.rb +0 -20
- data/lib/io_streams/delimited/writer.rb +0 -67
- data/test/csv_reader_test.rb +0 -34
- data/test/csv_writer_test.rb +0 -35
- data/test/delimited_reader_test.rb +0 -115
- data/test/delimited_writer_test.rb +0 -44
data/test/bzip2_writer_test.rb
CHANGED
@@ -1,43 +1,49 @@
|
|
1
1
|
require_relative 'test_helper'
|
2
2
|
|
3
|
-
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
@file_name = File.join(File.dirname(__FILE__), 'files', 'text.txt.gz')
|
9
|
-
@data = File.read(File.join(File.dirname(__FILE__), 'files', 'text.txt'))
|
10
|
-
end
|
3
|
+
class Bzip2WriterTest < Minitest::Test
|
4
|
+
describe IOStreams::Bzip2::Writer do
|
5
|
+
let :temp_file do
|
6
|
+
Tempfile.new('iostreams')
|
7
|
+
end
|
11
8
|
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
file_name = temp_file.to_path
|
16
|
-
IOStreams::Bzip2::Writer.open(file_name) do |io|
|
17
|
-
io.write(@data)
|
18
|
-
end
|
19
|
-
|
20
|
-
File.open(file_name, 'rb') do |file|
|
21
|
-
io = RBzip2.default_adapter::Decompressor.new(file)
|
22
|
-
result = io.read
|
23
|
-
temp_file.delete
|
24
|
-
assert_equal @data, result
|
25
|
-
end
|
26
|
-
end
|
9
|
+
let :file_name do
|
10
|
+
temp_file.path
|
11
|
+
end
|
27
12
|
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
13
|
+
let :decompressed do
|
14
|
+
File.read(File.join(File.dirname(__FILE__), 'files', 'text.txt'))
|
15
|
+
end
|
16
|
+
|
17
|
+
after do
|
18
|
+
temp_file.delete
|
19
|
+
end
|
33
20
|
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
21
|
+
describe '.open' do
|
22
|
+
it 'file' do
|
23
|
+
IOStreams::Bzip2::Writer.open(file_name) do |io|
|
24
|
+
io.write(decompressed)
|
25
|
+
end
|
26
|
+
|
27
|
+
File.open(file_name, 'rb') do |file|
|
28
|
+
io = RBzip2.default_adapter::Decompressor.new(file)
|
29
|
+
result = io.read
|
30
|
+
temp_file.delete
|
31
|
+
assert_equal decompressed, result
|
38
32
|
end
|
39
33
|
end
|
40
34
|
|
35
|
+
it 'stream' do
|
36
|
+
io_string = StringIO.new(''.b)
|
37
|
+
IOStreams::Bzip2::Writer.open(io_string) do |io|
|
38
|
+
io.write(decompressed)
|
39
|
+
end
|
40
|
+
|
41
|
+
io = StringIO.new(io_string.string)
|
42
|
+
rbzip2 = RBzip2.default_adapter::Decompressor.new(io)
|
43
|
+
data = rbzip2.read
|
44
|
+
assert_equal decompressed, data
|
45
|
+
end
|
41
46
|
end
|
47
|
+
|
42
48
|
end
|
43
49
|
end
|
data/test/file_reader_test.rb
CHANGED
@@ -1,32 +1,33 @@
|
|
1
1
|
require_relative 'test_helper'
|
2
2
|
|
3
|
-
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
@file_name = File.join(File.dirname(__FILE__), 'files', 'text.txt')
|
9
|
-
@data = File.read(@file_name)
|
10
|
-
end
|
3
|
+
class FileReaderTest < Minitest::Test
|
4
|
+
describe IOStreams::File::Reader do
|
5
|
+
let :file_name do
|
6
|
+
File.join(File.dirname(__FILE__), 'files', 'text.txt')
|
7
|
+
end
|
11
8
|
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
9
|
+
let :raw do
|
10
|
+
File.read(file_name)
|
11
|
+
end
|
12
|
+
|
13
|
+
describe '.open' do
|
14
|
+
it 'file' do
|
15
|
+
result = IOStreams::File::Reader.open(file_name) do |io|
|
16
|
+
io.read
|
18
17
|
end
|
18
|
+
assert_equal raw, result
|
19
|
+
end
|
19
20
|
|
20
|
-
|
21
|
-
|
21
|
+
it 'does not support streams' do
|
22
|
+
assert_raises ArgumentError do
|
23
|
+
File.open(file_name) do |file|
|
22
24
|
IOStreams::File::Reader.open(file) do |io|
|
23
25
|
io.read
|
24
26
|
end
|
25
27
|
end
|
26
|
-
assert_equal @data, result
|
27
28
|
end
|
28
29
|
end
|
29
|
-
|
30
30
|
end
|
31
|
+
|
31
32
|
end
|
32
33
|
end
|
data/test/file_writer_test.rb
CHANGED
@@ -1,34 +1,35 @@
|
|
1
1
|
require_relative 'test_helper'
|
2
2
|
|
3
|
-
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
@file_name = File.join(File.dirname(__FILE__), 'files', 'text.txt')
|
9
|
-
@data = File.read(@file_name)
|
10
|
-
end
|
3
|
+
class FileWriterTest < Minitest::Test
|
4
|
+
describe IOStreams::File::Writer do
|
5
|
+
let :file_name do
|
6
|
+
File.join(File.dirname(__FILE__), 'files', 'text.txt')
|
7
|
+
end
|
11
8
|
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
9
|
+
let :raw do
|
10
|
+
File.read(file_name)
|
11
|
+
end
|
12
|
+
|
13
|
+
describe '.open' do
|
14
|
+
it 'file' do
|
15
|
+
temp_file = Tempfile.new('rocket_job')
|
16
|
+
file_name = temp_file.to_path
|
17
|
+
IOStreams::File::Writer.open(file_name) do |io|
|
18
|
+
io.write(raw)
|
21
19
|
end
|
20
|
+
result = File.read(file_name)
|
21
|
+
assert_equal raw, result
|
22
|
+
end
|
22
23
|
|
23
|
-
|
24
|
-
|
24
|
+
it 'does not support streams' do
|
25
|
+
io_string = StringIO.new
|
26
|
+
assert_raises ArgumentError do
|
25
27
|
IOStreams::File::Writer.open(io_string) do |io|
|
26
|
-
io.write(
|
28
|
+
io.write(raw)
|
27
29
|
end
|
28
|
-
assert_equal @data, io_string.string
|
29
30
|
end
|
30
31
|
end
|
31
|
-
|
32
32
|
end
|
33
|
+
|
33
34
|
end
|
34
35
|
end
|
data/test/gzip_reader_test.rb
CHANGED
@@ -1,33 +1,32 @@
|
|
1
1
|
require_relative 'test_helper'
|
2
2
|
|
3
|
-
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
@file_name = File.join(File.dirname(__FILE__), 'files', 'text.txt.gz')
|
9
|
-
@gzip_data = File.open(@file_name, 'rb') { |f| f.read }
|
10
|
-
@data = Zlib::GzipReader.open(@file_name) { |gz| gz.read }
|
11
|
-
end
|
3
|
+
class GzipReaderTest < Minitest::Test
|
4
|
+
describe IOStreams::Gzip::Reader do
|
5
|
+
let :file_name do
|
6
|
+
File.join(File.dirname(__FILE__), 'files', 'text.txt.gz')
|
7
|
+
end
|
12
8
|
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
9
|
+
let :decompressed do
|
10
|
+
Zlib::GzipReader.open(file_name) { |gz| gz.read }
|
11
|
+
end
|
12
|
+
|
13
|
+
describe '.open' do
|
14
|
+
it 'file' do
|
15
|
+
result = IOStreams::Gzip::Reader.open(file_name) do |io|
|
16
|
+
io.read
|
19
17
|
end
|
18
|
+
assert_equal decompressed, result
|
19
|
+
end
|
20
20
|
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
end
|
21
|
+
it 'stream' do
|
22
|
+
result = File.open(file_name) do |file|
|
23
|
+
IOStreams::Gzip::Reader.open(file) do |io|
|
24
|
+
io.read
|
26
25
|
end
|
27
|
-
assert_equal @data, result
|
28
26
|
end
|
27
|
+
assert_equal decompressed, result
|
29
28
|
end
|
30
|
-
|
31
29
|
end
|
30
|
+
|
32
31
|
end
|
33
32
|
end
|
data/test/gzip_writer_test.rb
CHANGED
@@ -1,39 +1,45 @@
|
|
1
1
|
require_relative 'test_helper'
|
2
2
|
|
3
|
-
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
@file_name = File.join(File.dirname(__FILE__), 'files', 'text.txt.gz')
|
9
|
-
@data = Zlib::GzipReader.open(@file_name) { |gz| gz.read }
|
10
|
-
end
|
3
|
+
class GzipWriterTest < Minitest::Test
|
4
|
+
describe IOStreams::Gzip::Writer do
|
5
|
+
let :temp_file do
|
6
|
+
Tempfile.new('iostreams')
|
7
|
+
end
|
11
8
|
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
result = Zlib::GzipReader.open(file_name) { |gz| gz.read }
|
20
|
-
temp_file.delete
|
21
|
-
assert_equal @data, result
|
22
|
-
end
|
9
|
+
let :file_name do
|
10
|
+
temp_file.path
|
11
|
+
end
|
12
|
+
|
13
|
+
let :decompressed do
|
14
|
+
File.read(File.join(File.dirname(__FILE__), 'files', 'text.txt'))
|
15
|
+
end
|
23
16
|
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
gz.close
|
33
|
-
assert_equal @data, data
|
17
|
+
after do
|
18
|
+
temp_file.delete
|
19
|
+
end
|
20
|
+
|
21
|
+
describe '.open' do
|
22
|
+
it 'file' do
|
23
|
+
IOStreams::Gzip::Writer.open(file_name) do |io|
|
24
|
+
io.write(decompressed)
|
34
25
|
end
|
26
|
+
result = Zlib::GzipReader.open(file_name) { |gz| gz.read }
|
27
|
+
temp_file.delete
|
28
|
+
assert_equal decompressed, result
|
35
29
|
end
|
36
30
|
|
31
|
+
it 'stream' do
|
32
|
+
io_string = StringIO.new(''.b)
|
33
|
+
IOStreams::Gzip::Writer.open(io_string) do |io|
|
34
|
+
io.write(decompressed)
|
35
|
+
end
|
36
|
+
io = StringIO.new(io_string.string)
|
37
|
+
gz = Zlib::GzipReader.new(io)
|
38
|
+
data = gz.read
|
39
|
+
gz.close
|
40
|
+
assert_equal decompressed, data
|
41
|
+
end
|
37
42
|
end
|
43
|
+
|
38
44
|
end
|
39
45
|
end
|
data/test/io_streams_test.rb
CHANGED
@@ -1,84 +1,160 @@
|
|
1
1
|
require_relative 'test_helper'
|
2
2
|
|
3
|
-
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
@source_file_name = File.join(File.dirname(__FILE__), 'files', 'text.txt')
|
9
|
-
@data = File.read(@source_file_name)
|
10
|
-
|
11
|
-
@temp_file = Tempfile.new('iostreams')
|
12
|
-
@target_file_name = @temp_file.to_path
|
13
|
-
end
|
3
|
+
class IOStreamsTest < Minitest::Test
|
4
|
+
describe IOStreams do
|
5
|
+
let :source_file_name do
|
6
|
+
File.join(__dir__, 'files', 'text.txt')
|
7
|
+
end
|
14
8
|
|
15
|
-
|
16
|
-
|
17
|
-
|
9
|
+
let :data do
|
10
|
+
File.read(source_file_name)
|
11
|
+
end
|
18
12
|
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
IOStreams.writer(@target_file_name) do |target_stream|
|
23
|
-
IOStreams.copy(source_stream, target_stream)
|
24
|
-
end
|
25
|
-
end
|
26
|
-
actual = File.read(@target_file_name)
|
13
|
+
let :temp_file do
|
14
|
+
Tempfile.new('iostreams')
|
15
|
+
end
|
27
16
|
|
28
|
-
|
29
|
-
|
30
|
-
|
17
|
+
let :target_file_name do
|
18
|
+
temp_file.path
|
19
|
+
end
|
20
|
+
|
21
|
+
after do
|
22
|
+
temp_file.delete
|
23
|
+
end
|
31
24
|
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
25
|
+
describe '.copy' do
|
26
|
+
it 'streams' do
|
27
|
+
size = IOStreams.reader(source_file_name) do |source_stream|
|
28
|
+
IOStreams.writer(target_file_name) do |target_stream|
|
29
|
+
IOStreams.copy(source_stream, target_stream)
|
37
30
|
end
|
38
|
-
|
31
|
+
end
|
32
|
+
actual = File.read(target_file_name)
|
39
33
|
|
40
|
-
|
41
|
-
|
34
|
+
assert_equal actual, data
|
35
|
+
assert_equal actual.size, size
|
36
|
+
end
|
37
|
+
|
38
|
+
it 'IO stream' do
|
39
|
+
size = File.open(source_file_name) do |source_stream|
|
40
|
+
IOStreams.writer(target_file_name) do |target_stream|
|
41
|
+
IOStreams.copy(source_stream, target_stream)
|
42
|
+
end
|
42
43
|
end
|
44
|
+
actual = File.read(target_file_name)
|
45
|
+
|
46
|
+
assert_equal actual, data
|
47
|
+
assert_equal actual.size, size
|
43
48
|
end
|
44
49
|
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
actual = File.read(@target_file_name)
|
50
|
+
it 'files' do
|
51
|
+
size = IOStreams.copy(source_file_name, target_file_name)
|
52
|
+
actual = File.read(target_file_name)
|
49
53
|
|
50
|
-
|
51
|
-
|
52
|
-
end
|
54
|
+
assert_equal actual, data
|
55
|
+
assert_equal actual.size, size
|
53
56
|
end
|
57
|
+
end
|
54
58
|
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
|
59
|
-
|
59
|
+
describe '.streams_for_file_name' do
|
60
|
+
it 'file only' do
|
61
|
+
streams = IOStreams.streams_for_file_name('a.xyz')
|
62
|
+
assert_equal [], streams
|
63
|
+
end
|
60
64
|
|
61
|
-
|
62
|
-
|
63
|
-
|
64
|
-
|
65
|
+
it 'single stream' do
|
66
|
+
streams = IOStreams.streams_for_file_name('a.gz')
|
67
|
+
assert_equal [:gz], streams
|
68
|
+
end
|
65
69
|
|
66
|
-
|
67
|
-
|
68
|
-
|
69
|
-
|
70
|
+
it 'multiple streams' do
|
71
|
+
streams = IOStreams.streams_for_file_name('a.xlsx.gz')
|
72
|
+
assert_equal [:xlsx, :gz], streams
|
73
|
+
end
|
70
74
|
|
71
|
-
|
72
|
-
|
73
|
-
|
74
|
-
|
75
|
+
it 'is case-insensitive' do
|
76
|
+
streams = IOStreams.streams_for_file_name('a.GzIp')
|
77
|
+
assert_equal [:gzip], streams
|
78
|
+
end
|
75
79
|
|
76
|
-
|
77
|
-
|
78
|
-
|
79
|
-
|
80
|
+
it 'multiple streams are case-insensitive' do
|
81
|
+
streams = IOStreams.streams_for_file_name('a.XlsX.Gz')
|
82
|
+
assert_equal [:xlsx, :gz], streams
|
83
|
+
end
|
84
|
+
end
|
85
|
+
|
86
|
+
describe '.each_line' do
|
87
|
+
it 'returns a line at a time' do
|
88
|
+
lines = []
|
89
|
+
IOStreams.each_line(source_file_name) { |line| lines << line }
|
90
|
+
assert_equal data.lines.map(&:strip), lines
|
80
91
|
end
|
92
|
+
end
|
93
|
+
|
94
|
+
describe '.reader' do
|
95
|
+
# IOStreams.reader('abc.csv') do |io|
|
96
|
+
# p data while (data = io.read(128))
|
97
|
+
# end
|
98
|
+
end
|
81
99
|
|
100
|
+
describe '.each_line' do
|
101
|
+
# IOStreams.each_line('abc.csv') do |line|
|
102
|
+
# puts line
|
103
|
+
# end
|
82
104
|
end
|
105
|
+
|
106
|
+
describe '.each_row' do
|
107
|
+
# IOStreams.each_row('abc.csv') do |array|
|
108
|
+
# p array
|
109
|
+
# end
|
110
|
+
end
|
111
|
+
|
112
|
+
describe '.each_record' do
|
113
|
+
# IOStreams.each_record('abc.csv') do |hash|
|
114
|
+
# p hash
|
115
|
+
# end
|
116
|
+
|
117
|
+
# array = [
|
118
|
+
# 'name, address, zip_code',
|
119
|
+
# 'Jack, Down Under, 12345'
|
120
|
+
# ]
|
121
|
+
# IOStreams.each_record(array) do |hash|
|
122
|
+
# p hash
|
123
|
+
# end
|
124
|
+
end
|
125
|
+
|
126
|
+
describe '.writer' do
|
127
|
+
# IOStreams.writer('abc.csv') do |io|
|
128
|
+
# io.write('This')
|
129
|
+
# io.write(' is ')
|
130
|
+
# io.write(" one line\n")
|
131
|
+
# end
|
132
|
+
end
|
133
|
+
|
134
|
+
describe '.line_writer' do
|
135
|
+
# IOStreams.line_writer('abc.csv') do |file|
|
136
|
+
# file << 'these'
|
137
|
+
# file << 'are'
|
138
|
+
# file << 'all'
|
139
|
+
# file << 'separate'
|
140
|
+
# file << 'lines'
|
141
|
+
# end
|
142
|
+
end
|
143
|
+
|
144
|
+
describe '.row_writer' do
|
145
|
+
# IOStreams.row_writer('abc.csv') do |io|
|
146
|
+
# io << %w[name address zip_code]
|
147
|
+
# io << %w[Jack There 1234]
|
148
|
+
# io << ['Joe', 'Over There somewhere', 1234]
|
149
|
+
# end
|
150
|
+
end
|
151
|
+
|
152
|
+
describe '.record_writer' do
|
153
|
+
# IOStreams.record_writer('abc.csv') do |stream|
|
154
|
+
# stream << {name: 'Jack', address: 'There', zip_code: 1234}
|
155
|
+
# stream << {name: 'Joe', address: 'Over There somewhere', zip_code: 1234}
|
156
|
+
# end
|
157
|
+
end
|
158
|
+
|
83
159
|
end
|
84
160
|
end
|