iostreams 0.14.0 → 0.15.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (62) hide show
  1. checksums.yaml +4 -4
  2. data/LICENSE +202 -0
  3. data/README.md +155 -47
  4. data/lib/io_streams/file/reader.rb +7 -8
  5. data/lib/io_streams/file/writer.rb +7 -8
  6. data/lib/io_streams/io_streams.rb +313 -129
  7. data/lib/io_streams/{delimited → line}/reader.rb +20 -30
  8. data/lib/io_streams/line/writer.rb +81 -0
  9. data/lib/io_streams/pgp.rb +4 -14
  10. data/lib/io_streams/record/reader.rb +55 -0
  11. data/lib/io_streams/record/writer.rb +63 -0
  12. data/lib/io_streams/row/reader.rb +60 -0
  13. data/lib/io_streams/row/writer.rb +62 -0
  14. data/lib/io_streams/s3.rb +25 -0
  15. data/lib/io_streams/s3/reader.rb +64 -0
  16. data/lib/io_streams/s3/writer.rb +13 -0
  17. data/lib/io_streams/streams.rb +1 -1
  18. data/lib/io_streams/tabular.rb +163 -0
  19. data/lib/io_streams/tabular/errors.rb +14 -0
  20. data/lib/io_streams/tabular/header.rb +146 -0
  21. data/lib/io_streams/tabular/parser/array.rb +26 -0
  22. data/lib/io_streams/tabular/parser/base.rb +12 -0
  23. data/lib/io_streams/tabular/parser/csv.rb +35 -0
  24. data/lib/io_streams/tabular/parser/fixed.rb +88 -0
  25. data/lib/io_streams/tabular/parser/hash.rb +21 -0
  26. data/lib/io_streams/tabular/parser/json.rb +25 -0
  27. data/lib/io_streams/tabular/parser/psv.rb +34 -0
  28. data/lib/io_streams/tabular/utility/csv_row.rb +115 -0
  29. data/lib/io_streams/version.rb +2 -2
  30. data/lib/io_streams/xlsx/reader.rb +1 -1
  31. data/lib/io_streams/zip/reader.rb +1 -1
  32. data/lib/io_streams/zip/writer.rb +1 -1
  33. data/lib/iostreams.rb +21 -10
  34. data/test/bzip2_reader_test.rb +21 -22
  35. data/test/bzip2_writer_test.rb +38 -32
  36. data/test/file_reader_test.rb +19 -18
  37. data/test/file_writer_test.rb +23 -22
  38. data/test/files/test.json +3 -0
  39. data/test/gzip_reader_test.rb +21 -22
  40. data/test/gzip_writer_test.rb +35 -29
  41. data/test/io_streams_test.rb +137 -61
  42. data/test/line_reader_test.rb +105 -0
  43. data/test/line_writer_test.rb +50 -0
  44. data/test/pgp_reader_test.rb +29 -29
  45. data/test/pgp_test.rb +149 -195
  46. data/test/pgp_writer_test.rb +63 -62
  47. data/test/record_reader_test.rb +61 -0
  48. data/test/record_writer_test.rb +73 -0
  49. data/test/row_reader_test.rb +34 -0
  50. data/test/row_writer_test.rb +51 -0
  51. data/test/tabular_test.rb +184 -0
  52. data/test/xlsx_reader_test.rb +13 -17
  53. data/test/zip_reader_test.rb +21 -22
  54. data/test/zip_writer_test.rb +40 -36
  55. metadata +41 -17
  56. data/lib/io_streams/csv/reader.rb +0 -21
  57. data/lib/io_streams/csv/writer.rb +0 -20
  58. data/lib/io_streams/delimited/writer.rb +0 -67
  59. data/test/csv_reader_test.rb +0 -34
  60. data/test/csv_writer_test.rb +0 -35
  61. data/test/delimited_reader_test.rb +0 -115
  62. data/test/delimited_writer_test.rb +0 -44
@@ -1,43 +1,49 @@
1
1
  require_relative 'test_helper'
2
2
 
3
- # Unit Test for IOStreams::Bzip2
4
- module Streams
5
- class Bzip2WriterTest < Minitest::Test
6
- describe IOStreams::Bzip2::Writer do
7
- before do
8
- @file_name = File.join(File.dirname(__FILE__), 'files', 'text.txt.gz')
9
- @data = File.read(File.join(File.dirname(__FILE__), 'files', 'text.txt'))
10
- end
3
+ class Bzip2WriterTest < Minitest::Test
4
+ describe IOStreams::Bzip2::Writer do
5
+ let :temp_file do
6
+ Tempfile.new('iostreams')
7
+ end
11
8
 
12
- describe '.open' do
13
- it 'file' do
14
- temp_file = Tempfile.new('rocket_job')
15
- file_name = temp_file.to_path
16
- IOStreams::Bzip2::Writer.open(file_name) do |io|
17
- io.write(@data)
18
- end
19
-
20
- File.open(file_name, 'rb') do |file|
21
- io = RBzip2.default_adapter::Decompressor.new(file)
22
- result = io.read
23
- temp_file.delete
24
- assert_equal @data, result
25
- end
26
- end
9
+ let :file_name do
10
+ temp_file.path
11
+ end
27
12
 
28
- it 'stream' do
29
- io_string = StringIO.new(''.force_encoding('ASCII-8BIT'))
30
- IOStreams::Bzip2::Writer.open(io_string) do |io|
31
- io.write(@data)
32
- end
13
+ let :decompressed do
14
+ File.read(File.join(File.dirname(__FILE__), 'files', 'text.txt'))
15
+ end
16
+
17
+ after do
18
+ temp_file.delete
19
+ end
33
20
 
34
- io = StringIO.new(io_string.string)
35
- rbzip2 = RBzip2.default_adapter::Decompressor.new(io)
36
- data = rbzip2.read
37
- assert_equal @data, data
21
+ describe '.open' do
22
+ it 'file' do
23
+ IOStreams::Bzip2::Writer.open(file_name) do |io|
24
+ io.write(decompressed)
25
+ end
26
+
27
+ File.open(file_name, 'rb') do |file|
28
+ io = RBzip2.default_adapter::Decompressor.new(file)
29
+ result = io.read
30
+ temp_file.delete
31
+ assert_equal decompressed, result
38
32
  end
39
33
  end
40
34
 
35
+ it 'stream' do
36
+ io_string = StringIO.new(''.b)
37
+ IOStreams::Bzip2::Writer.open(io_string) do |io|
38
+ io.write(decompressed)
39
+ end
40
+
41
+ io = StringIO.new(io_string.string)
42
+ rbzip2 = RBzip2.default_adapter::Decompressor.new(io)
43
+ data = rbzip2.read
44
+ assert_equal decompressed, data
45
+ end
41
46
  end
47
+
42
48
  end
43
49
  end
@@ -1,32 +1,33 @@
1
1
  require_relative 'test_helper'
2
2
 
3
- # Unit Test for IOStreams::File
4
- module Streams
5
- class FileReaderTest < Minitest::Test
6
- describe IOStreams::File::Reader do
7
- before do
8
- @file_name = File.join(File.dirname(__FILE__), 'files', 'text.txt')
9
- @data = File.read(@file_name)
10
- end
3
+ class FileReaderTest < Minitest::Test
4
+ describe IOStreams::File::Reader do
5
+ let :file_name do
6
+ File.join(File.dirname(__FILE__), 'files', 'text.txt')
7
+ end
11
8
 
12
- describe '.open' do
13
- it 'file' do
14
- result = IOStreams::File::Reader.open(@file_name) do |io|
15
- io.read
16
- end
17
- assert_equal @data, result
9
+ let :raw do
10
+ File.read(file_name)
11
+ end
12
+
13
+ describe '.open' do
14
+ it 'file' do
15
+ result = IOStreams::File::Reader.open(file_name) do |io|
16
+ io.read
18
17
  end
18
+ assert_equal raw, result
19
+ end
19
20
 
20
- it 'stream' do
21
- result = File.open(@file_name) do |file|
21
+ it 'does not support streams' do
22
+ assert_raises ArgumentError do
23
+ File.open(file_name) do |file|
22
24
  IOStreams::File::Reader.open(file) do |io|
23
25
  io.read
24
26
  end
25
27
  end
26
- assert_equal @data, result
27
28
  end
28
29
  end
29
-
30
30
  end
31
+
31
32
  end
32
33
  end
@@ -1,34 +1,35 @@
1
1
  require_relative 'test_helper'
2
2
 
3
- # Unit Test for IOStreams::File
4
- module Streams
5
- class FileWriterTest < Minitest::Test
6
- describe IOStreams::File::Writer do
7
- before do
8
- @file_name = File.join(File.dirname(__FILE__), 'files', 'text.txt')
9
- @data = File.read(@file_name)
10
- end
3
+ class FileWriterTest < Minitest::Test
4
+ describe IOStreams::File::Writer do
5
+ let :file_name do
6
+ File.join(File.dirname(__FILE__), 'files', 'text.txt')
7
+ end
11
8
 
12
- describe '.open' do
13
- it 'file' do
14
- temp_file = Tempfile.new('rocket_job')
15
- file_name = temp_file.to_path
16
- IOStreams::File::Writer.open(file_name) do |io|
17
- io.write(@data)
18
- end
19
- result = File.read(file_name)
20
- assert_equal @data, result
9
+ let :raw do
10
+ File.read(file_name)
11
+ end
12
+
13
+ describe '.open' do
14
+ it 'file' do
15
+ temp_file = Tempfile.new('rocket_job')
16
+ file_name = temp_file.to_path
17
+ IOStreams::File::Writer.open(file_name) do |io|
18
+ io.write(raw)
21
19
  end
20
+ result = File.read(file_name)
21
+ assert_equal raw, result
22
+ end
22
23
 
23
- it 'stream' do
24
- io_string = StringIO.new
24
+ it 'does not support streams' do
25
+ io_string = StringIO.new
26
+ assert_raises ArgumentError do
25
27
  IOStreams::File::Writer.open(io_string) do |io|
26
- io.write(@data)
28
+ io.write(raw)
27
29
  end
28
- assert_equal @data, io_string.string
29
30
  end
30
31
  end
31
-
32
32
  end
33
+
33
34
  end
34
35
  end
@@ -0,0 +1,3 @@
1
+ {"name":"Joe Black"," address":" 23 Where Rd"," state":" FL"," zip":" 12452"}
2
+ {"name":"Who is this"," address":" 42 Able Rd"," state":" GA"," zip":" 23432"}
3
+ {"name":"Another's Person"," address":" 42 and 2nd St"," state":" NY"," zip":" 17449"}
@@ -1,33 +1,32 @@
1
1
  require_relative 'test_helper'
2
2
 
3
- # Unit Test for IOStreams::Gzip
4
- module Streams
5
- class GzipReaderTest < Minitest::Test
6
- describe IOStreams::Gzip::Reader do
7
- before do
8
- @file_name = File.join(File.dirname(__FILE__), 'files', 'text.txt.gz')
9
- @gzip_data = File.open(@file_name, 'rb') { |f| f.read }
10
- @data = Zlib::GzipReader.open(@file_name) { |gz| gz.read }
11
- end
3
+ class GzipReaderTest < Minitest::Test
4
+ describe IOStreams::Gzip::Reader do
5
+ let :file_name do
6
+ File.join(File.dirname(__FILE__), 'files', 'text.txt.gz')
7
+ end
12
8
 
13
- describe '.open' do
14
- it 'file' do
15
- result = IOStreams::Gzip::Reader.open(@file_name) do |io|
16
- io.read
17
- end
18
- assert_equal @data, result
9
+ let :decompressed do
10
+ Zlib::GzipReader.open(file_name) { |gz| gz.read }
11
+ end
12
+
13
+ describe '.open' do
14
+ it 'file' do
15
+ result = IOStreams::Gzip::Reader.open(file_name) do |io|
16
+ io.read
19
17
  end
18
+ assert_equal decompressed, result
19
+ end
20
20
 
21
- it 'stream' do
22
- result = File.open(@file_name) do |file|
23
- IOStreams::Gzip::Reader.open(file) do |io|
24
- io.read
25
- end
21
+ it 'stream' do
22
+ result = File.open(file_name) do |file|
23
+ IOStreams::Gzip::Reader.open(file) do |io|
24
+ io.read
26
25
  end
27
- assert_equal @data, result
28
26
  end
27
+ assert_equal decompressed, result
29
28
  end
30
-
31
29
  end
30
+
32
31
  end
33
32
  end
@@ -1,39 +1,45 @@
1
1
  require_relative 'test_helper'
2
2
 
3
- # Unit Test for IOStreams::Gzip
4
- module Streams
5
- class GzipWriterTest < Minitest::Test
6
- describe IOStreams::Gzip::Writer do
7
- before do
8
- @file_name = File.join(File.dirname(__FILE__), 'files', 'text.txt.gz')
9
- @data = Zlib::GzipReader.open(@file_name) { |gz| gz.read }
10
- end
3
+ class GzipWriterTest < Minitest::Test
4
+ describe IOStreams::Gzip::Writer do
5
+ let :temp_file do
6
+ Tempfile.new('iostreams')
7
+ end
11
8
 
12
- describe '.open' do
13
- it 'file' do
14
- temp_file = Tempfile.new('rocket_job')
15
- file_name = temp_file.to_path
16
- IOStreams::Gzip::Writer.open(file_name) do |io|
17
- io.write(@data)
18
- end
19
- result = Zlib::GzipReader.open(file_name) { |gz| gz.read }
20
- temp_file.delete
21
- assert_equal @data, result
22
- end
9
+ let :file_name do
10
+ temp_file.path
11
+ end
12
+
13
+ let :decompressed do
14
+ File.read(File.join(File.dirname(__FILE__), 'files', 'text.txt'))
15
+ end
23
16
 
24
- it 'stream' do
25
- io_string = StringIO.new(''.force_encoding('ASCII-8BIT'))
26
- IOStreams::Gzip::Writer.open(io_string) do |io|
27
- io.write(@data)
28
- end
29
- io = StringIO.new(io_string.string)
30
- gz = Zlib::GzipReader.new(io)
31
- data = gz.read
32
- gz.close
33
- assert_equal @data, data
17
+ after do
18
+ temp_file.delete
19
+ end
20
+
21
+ describe '.open' do
22
+ it 'file' do
23
+ IOStreams::Gzip::Writer.open(file_name) do |io|
24
+ io.write(decompressed)
34
25
  end
26
+ result = Zlib::GzipReader.open(file_name) { |gz| gz.read }
27
+ temp_file.delete
28
+ assert_equal decompressed, result
35
29
  end
36
30
 
31
+ it 'stream' do
32
+ io_string = StringIO.new(''.b)
33
+ IOStreams::Gzip::Writer.open(io_string) do |io|
34
+ io.write(decompressed)
35
+ end
36
+ io = StringIO.new(io_string.string)
37
+ gz = Zlib::GzipReader.new(io)
38
+ data = gz.read
39
+ gz.close
40
+ assert_equal decompressed, data
41
+ end
37
42
  end
43
+
38
44
  end
39
45
  end
@@ -1,84 +1,160 @@
1
1
  require_relative 'test_helper'
2
2
 
3
- # Unit Test for IOStreams::File
4
- module Streams
5
- class IOStreamsTest < Minitest::Test
6
- describe IOStreams do
7
- before do
8
- @source_file_name = File.join(File.dirname(__FILE__), 'files', 'text.txt')
9
- @data = File.read(@source_file_name)
10
-
11
- @temp_file = Tempfile.new('iostreams')
12
- @target_file_name = @temp_file.to_path
13
- end
3
+ class IOStreamsTest < Minitest::Test
4
+ describe IOStreams do
5
+ let :source_file_name do
6
+ File.join(__dir__, 'files', 'text.txt')
7
+ end
14
8
 
15
- after do
16
- @temp_file.delete if @temp_file
17
- end
9
+ let :data do
10
+ File.read(source_file_name)
11
+ end
18
12
 
19
- describe '.copy' do
20
- it 'file' do
21
- size = IOStreams.reader(@source_file_name) do |source_stream|
22
- IOStreams.writer(@target_file_name) do |target_stream|
23
- IOStreams.copy(source_stream, target_stream)
24
- end
25
- end
26
- actual = File.read(@target_file_name)
13
+ let :temp_file do
14
+ Tempfile.new('iostreams')
15
+ end
27
16
 
28
- assert_equal actual, @data
29
- assert_equal actual.size, size
30
- end
17
+ let :target_file_name do
18
+ temp_file.path
19
+ end
20
+
21
+ after do
22
+ temp_file.delete
23
+ end
31
24
 
32
- it 'stream' do
33
- size = File.open(@source_file_name) do |source_stream|
34
- IOStreams.writer(@target_file_name) do |target_stream|
35
- IOStreams.copy(source_stream, target_stream)
36
- end
25
+ describe '.copy' do
26
+ it 'streams' do
27
+ size = IOStreams.reader(source_file_name) do |source_stream|
28
+ IOStreams.writer(target_file_name) do |target_stream|
29
+ IOStreams.copy(source_stream, target_stream)
37
30
  end
38
- actual = File.read(@target_file_name)
31
+ end
32
+ actual = File.read(target_file_name)
39
33
 
40
- assert_equal actual, @data
41
- assert_equal actual.size, size
34
+ assert_equal actual, data
35
+ assert_equal actual.size, size
36
+ end
37
+
38
+ it 'IO stream' do
39
+ size = File.open(source_file_name) do |source_stream|
40
+ IOStreams.writer(target_file_name) do |target_stream|
41
+ IOStreams.copy(source_stream, target_stream)
42
+ end
42
43
  end
44
+ actual = File.read(target_file_name)
45
+
46
+ assert_equal actual, data
47
+ assert_equal actual.size, size
43
48
  end
44
49
 
45
- describe '.copy_file' do
46
- it 'copies' do
47
- size = IOStreams.copy_file(@source_file_name, @target_file_name)
48
- actual = File.read(@target_file_name)
50
+ it 'files' do
51
+ size = IOStreams.copy(source_file_name, target_file_name)
52
+ actual = File.read(target_file_name)
49
53
 
50
- assert_equal actual, @data
51
- assert_equal actual.size, size
52
- end
54
+ assert_equal actual, data
55
+ assert_equal actual.size, size
53
56
  end
57
+ end
54
58
 
55
- describe '.streams_for_file_name' do
56
- it 'file only' do
57
- streams = IOStreams.streams_for_file_name('a.xyz')
58
- assert_equal [:file], streams
59
- end
59
+ describe '.streams_for_file_name' do
60
+ it 'file only' do
61
+ streams = IOStreams.streams_for_file_name('a.xyz')
62
+ assert_equal [], streams
63
+ end
60
64
 
61
- it 'single stream' do
62
- streams = IOStreams.streams_for_file_name('a.gz')
63
- assert_equal [:gz], streams
64
- end
65
+ it 'single stream' do
66
+ streams = IOStreams.streams_for_file_name('a.gz')
67
+ assert_equal [:gz], streams
68
+ end
65
69
 
66
- it 'multiple streams' do
67
- streams = IOStreams.streams_for_file_name('a.xlsx.gz')
68
- assert_equal [:xlsx, :gz], streams
69
- end
70
+ it 'multiple streams' do
71
+ streams = IOStreams.streams_for_file_name('a.xlsx.gz')
72
+ assert_equal [:xlsx, :gz], streams
73
+ end
70
74
 
71
- it 'is case-insensitive' do
72
- streams = IOStreams.streams_for_file_name('a.GzIp')
73
- assert_equal [:gzip], streams
74
- end
75
+ it 'is case-insensitive' do
76
+ streams = IOStreams.streams_for_file_name('a.GzIp')
77
+ assert_equal [:gzip], streams
78
+ end
75
79
 
76
- it 'multiple streams are case-insensitive' do
77
- streams = IOStreams.streams_for_file_name('a.XlsX.Gz')
78
- assert_equal [:xlsx, :gz], streams
79
- end
80
+ it 'multiple streams are case-insensitive' do
81
+ streams = IOStreams.streams_for_file_name('a.XlsX.Gz')
82
+ assert_equal [:xlsx, :gz], streams
83
+ end
84
+ end
85
+
86
+ describe '.each_line' do
87
+ it 'returns a line at a time' do
88
+ lines = []
89
+ IOStreams.each_line(source_file_name) { |line| lines << line }
90
+ assert_equal data.lines.map(&:strip), lines
80
91
  end
92
+ end
93
+
94
+ describe '.reader' do
95
+ # IOStreams.reader('abc.csv') do |io|
96
+ # p data while (data = io.read(128))
97
+ # end
98
+ end
81
99
 
100
+ describe '.each_line' do
101
+ # IOStreams.each_line('abc.csv') do |line|
102
+ # puts line
103
+ # end
82
104
  end
105
+
106
+ describe '.each_row' do
107
+ # IOStreams.each_row('abc.csv') do |array|
108
+ # p array
109
+ # end
110
+ end
111
+
112
+ describe '.each_record' do
113
+ # IOStreams.each_record('abc.csv') do |hash|
114
+ # p hash
115
+ # end
116
+
117
+ # array = [
118
+ # 'name, address, zip_code',
119
+ # 'Jack, Down Under, 12345'
120
+ # ]
121
+ # IOStreams.each_record(array) do |hash|
122
+ # p hash
123
+ # end
124
+ end
125
+
126
+ describe '.writer' do
127
+ # IOStreams.writer('abc.csv') do |io|
128
+ # io.write('This')
129
+ # io.write(' is ')
130
+ # io.write(" one line\n")
131
+ # end
132
+ end
133
+
134
+ describe '.line_writer' do
135
+ # IOStreams.line_writer('abc.csv') do |file|
136
+ # file << 'these'
137
+ # file << 'are'
138
+ # file << 'all'
139
+ # file << 'separate'
140
+ # file << 'lines'
141
+ # end
142
+ end
143
+
144
+ describe '.row_writer' do
145
+ # IOStreams.row_writer('abc.csv') do |io|
146
+ # io << %w[name address zip_code]
147
+ # io << %w[Jack There 1234]
148
+ # io << ['Joe', 'Over There somewhere', 1234]
149
+ # end
150
+ end
151
+
152
+ describe '.record_writer' do
153
+ # IOStreams.record_writer('abc.csv') do |stream|
154
+ # stream << {name: 'Jack', address: 'There', zip_code: 1234}
155
+ # stream << {name: 'Joe', address: 'Over There somewhere', zip_code: 1234}
156
+ # end
157
+ end
158
+
83
159
  end
84
160
  end