iostreams 0.14.0 → 0.15.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/LICENSE +202 -0
- data/README.md +155 -47
- data/lib/io_streams/file/reader.rb +7 -8
- data/lib/io_streams/file/writer.rb +7 -8
- data/lib/io_streams/io_streams.rb +313 -129
- data/lib/io_streams/{delimited → line}/reader.rb +20 -30
- data/lib/io_streams/line/writer.rb +81 -0
- data/lib/io_streams/pgp.rb +4 -14
- data/lib/io_streams/record/reader.rb +55 -0
- data/lib/io_streams/record/writer.rb +63 -0
- data/lib/io_streams/row/reader.rb +60 -0
- data/lib/io_streams/row/writer.rb +62 -0
- data/lib/io_streams/s3.rb +25 -0
- data/lib/io_streams/s3/reader.rb +64 -0
- data/lib/io_streams/s3/writer.rb +13 -0
- data/lib/io_streams/streams.rb +1 -1
- data/lib/io_streams/tabular.rb +163 -0
- data/lib/io_streams/tabular/errors.rb +14 -0
- data/lib/io_streams/tabular/header.rb +146 -0
- data/lib/io_streams/tabular/parser/array.rb +26 -0
- data/lib/io_streams/tabular/parser/base.rb +12 -0
- data/lib/io_streams/tabular/parser/csv.rb +35 -0
- data/lib/io_streams/tabular/parser/fixed.rb +88 -0
- data/lib/io_streams/tabular/parser/hash.rb +21 -0
- data/lib/io_streams/tabular/parser/json.rb +25 -0
- data/lib/io_streams/tabular/parser/psv.rb +34 -0
- data/lib/io_streams/tabular/utility/csv_row.rb +115 -0
- data/lib/io_streams/version.rb +2 -2
- data/lib/io_streams/xlsx/reader.rb +1 -1
- data/lib/io_streams/zip/reader.rb +1 -1
- data/lib/io_streams/zip/writer.rb +1 -1
- data/lib/iostreams.rb +21 -10
- data/test/bzip2_reader_test.rb +21 -22
- data/test/bzip2_writer_test.rb +38 -32
- data/test/file_reader_test.rb +19 -18
- data/test/file_writer_test.rb +23 -22
- data/test/files/test.json +3 -0
- data/test/gzip_reader_test.rb +21 -22
- data/test/gzip_writer_test.rb +35 -29
- data/test/io_streams_test.rb +137 -61
- data/test/line_reader_test.rb +105 -0
- data/test/line_writer_test.rb +50 -0
- data/test/pgp_reader_test.rb +29 -29
- data/test/pgp_test.rb +149 -195
- data/test/pgp_writer_test.rb +63 -62
- data/test/record_reader_test.rb +61 -0
- data/test/record_writer_test.rb +73 -0
- data/test/row_reader_test.rb +34 -0
- data/test/row_writer_test.rb +51 -0
- data/test/tabular_test.rb +184 -0
- data/test/xlsx_reader_test.rb +13 -17
- data/test/zip_reader_test.rb +21 -22
- data/test/zip_writer_test.rb +40 -36
- metadata +41 -17
- data/lib/io_streams/csv/reader.rb +0 -21
- data/lib/io_streams/csv/writer.rb +0 -20
- data/lib/io_streams/delimited/writer.rb +0 -67
- data/test/csv_reader_test.rb +0 -34
- data/test/csv_writer_test.rb +0 -35
- data/test/delimited_reader_test.rb +0 -115
- data/test/delimited_writer_test.rb +0 -44
data/test/csv_reader_test.rb
DELETED
@@ -1,34 +0,0 @@
|
|
1
|
-
require_relative 'test_helper'
|
2
|
-
|
3
|
-
# Unit Test for IOStreams::File
|
4
|
-
module Streams
|
5
|
-
class CSVReaderTest < Minitest::Test
|
6
|
-
describe IOStreams::CSV::Reader do
|
7
|
-
before do
|
8
|
-
@file_name = File.join(File.dirname(__FILE__), 'files', 'test.csv')
|
9
|
-
@data = CSV.read(@file_name)
|
10
|
-
end
|
11
|
-
|
12
|
-
describe '.open' do
|
13
|
-
it 'file' do
|
14
|
-
rows = []
|
15
|
-
IOStreams::CSV::Reader.open(@file_name) do |io|
|
16
|
-
io.each { |row| rows << row }
|
17
|
-
end
|
18
|
-
assert_equal @data, rows
|
19
|
-
end
|
20
|
-
|
21
|
-
it 'stream' do
|
22
|
-
rows = []
|
23
|
-
File.open(@file_name) do |file|
|
24
|
-
IOStreams::CSV::Reader.open(file) do |io|
|
25
|
-
io.each { |row| rows << row }
|
26
|
-
end
|
27
|
-
end
|
28
|
-
assert_equal @data, rows
|
29
|
-
end
|
30
|
-
end
|
31
|
-
|
32
|
-
end
|
33
|
-
end
|
34
|
-
end
|
data/test/csv_writer_test.rb
DELETED
@@ -1,35 +0,0 @@
|
|
1
|
-
require_relative 'test_helper'
|
2
|
-
require 'csv'
|
3
|
-
|
4
|
-
module Streams
|
5
|
-
class CSVWriterTest < Minitest::Test
|
6
|
-
describe IOStreams::CSV::Writer do
|
7
|
-
before do
|
8
|
-
@file_name = File.join(File.dirname(__FILE__), 'files', 'test.csv')
|
9
|
-
@data = ::CSV.read(@file_name)
|
10
|
-
@raw_csv_data = ::File.read(@file_name)
|
11
|
-
end
|
12
|
-
|
13
|
-
describe '.open' do
|
14
|
-
it 'file' do
|
15
|
-
temp_file = Tempfile.new('rocket_job')
|
16
|
-
file_name = temp_file.to_path
|
17
|
-
IOStreams::CSV::Writer.open(file_name) do |io|
|
18
|
-
@data.each { |row| io << row }
|
19
|
-
end
|
20
|
-
result = File.read(file_name)
|
21
|
-
assert_equal @raw_csv_data, result
|
22
|
-
end
|
23
|
-
|
24
|
-
it 'stream' do
|
25
|
-
io_string = StringIO.new
|
26
|
-
IOStreams::CSV::Writer.open(io_string) do |io|
|
27
|
-
@data.each { |row| io << row }
|
28
|
-
end
|
29
|
-
assert_equal @raw_csv_data, io_string.string
|
30
|
-
end
|
31
|
-
end
|
32
|
-
|
33
|
-
end
|
34
|
-
end
|
35
|
-
end
|
@@ -1,115 +0,0 @@
|
|
1
|
-
require_relative 'test_helper'
|
2
|
-
|
3
|
-
# Unit Test for IOStreams::File
|
4
|
-
module Streams
|
5
|
-
class DelimitedReaderTest < Minitest::Test
|
6
|
-
describe IOStreams::Delimited::Reader do
|
7
|
-
before do
|
8
|
-
@file_name = File.join(File.dirname(__FILE__), 'files', 'text.txt')
|
9
|
-
@data = []
|
10
|
-
File.open(@file_name, 'rt') do |file|
|
11
|
-
while !file.eof?
|
12
|
-
@data << file.readline.strip
|
13
|
-
end
|
14
|
-
end
|
15
|
-
end
|
16
|
-
|
17
|
-
describe '#initialize' do
|
18
|
-
it 'does not strip invalid characters' do
|
19
|
-
bad_lines = [
|
20
|
-
"New M\xE9xico,NE",
|
21
|
-
'good line',
|
22
|
-
"New M\xE9xico,SF"
|
23
|
-
]
|
24
|
-
input = StringIO.new(bad_lines.join("\n"))
|
25
|
-
lines = []
|
26
|
-
IOStreams::Delimited::Reader.open(input) do |io|
|
27
|
-
assert_equal false, io.strip_non_printable
|
28
|
-
assert_raises ArgumentError do
|
29
|
-
io.each { |line| lines << line }
|
30
|
-
end
|
31
|
-
end
|
32
|
-
end
|
33
|
-
|
34
|
-
it 'strips invalid characters' do
|
35
|
-
bad_lines = [
|
36
|
-
"New M\xE9xico,NE",
|
37
|
-
'good line',
|
38
|
-
"New M\xE9xico,SF"
|
39
|
-
]
|
40
|
-
fixed_lines = bad_lines.collect { |line| line.force_encoding('BINARY').gsub(/[^[:print:]|\r|\n]/, '') }
|
41
|
-
input = StringIO.new(bad_lines.join("\n"))
|
42
|
-
lines = []
|
43
|
-
IOStreams::Delimited::Reader.open(input, strip_non_printable: true) do |io|
|
44
|
-
assert_equal true, io.strip_non_printable
|
45
|
-
io.each { |line| lines << line }
|
46
|
-
end
|
47
|
-
assert_equal fixed_lines, lines
|
48
|
-
end
|
49
|
-
end
|
50
|
-
|
51
|
-
describe '#each' do
|
52
|
-
it 'each_line file' do
|
53
|
-
lines = []
|
54
|
-
IOStreams::Delimited::Reader.open(@file_name) do |io|
|
55
|
-
io.each { |line| lines << line }
|
56
|
-
end
|
57
|
-
assert_equal @data, lines
|
58
|
-
end
|
59
|
-
|
60
|
-
it 'each_line stream' do
|
61
|
-
lines = []
|
62
|
-
File.open(@file_name) do |file|
|
63
|
-
IOStreams::Delimited::Reader.open(file) do |io|
|
64
|
-
io.each { |line| lines << line }
|
65
|
-
end
|
66
|
-
end
|
67
|
-
assert_equal @data, lines
|
68
|
-
end
|
69
|
-
|
70
|
-
["\r\n", "\n\r", "\n", "\r"].each do |delimiter|
|
71
|
-
it "autodetect delimiter: #{delimiter.inspect}" do
|
72
|
-
lines = []
|
73
|
-
stream = StringIO.new(@data.join(delimiter))
|
74
|
-
IOStreams::Delimited::Reader.open(stream, buffer_size: 15) do |io|
|
75
|
-
io.each { |line| lines << line }
|
76
|
-
end
|
77
|
-
assert_equal @data, lines
|
78
|
-
end
|
79
|
-
end
|
80
|
-
|
81
|
-
['@', 'BLAH'].each do |delimiter|
|
82
|
-
it "reads delimited #{delimiter.inspect}" do
|
83
|
-
lines = []
|
84
|
-
stream = StringIO.new(@data.join(delimiter))
|
85
|
-
IOStreams::Delimited::Reader.open(stream, buffer_size: 15, delimiter: delimiter) do |io|
|
86
|
-
io.each { |line| lines << line }
|
87
|
-
end
|
88
|
-
assert_equal @data, lines
|
89
|
-
end
|
90
|
-
end
|
91
|
-
|
92
|
-
it 'reads binary delimited' do
|
93
|
-
delimiter = "\x01"
|
94
|
-
lines = []
|
95
|
-
stream = StringIO.new(@data.join(delimiter))
|
96
|
-
IOStreams::Delimited::Reader.open(stream, buffer_size: 15, delimiter: delimiter, encoding: IOStreams::BINARY_ENCODING) do |io|
|
97
|
-
io.each { |line| lines << line }
|
98
|
-
end
|
99
|
-
assert_equal @data, lines
|
100
|
-
end
|
101
|
-
end
|
102
|
-
|
103
|
-
describe '.read' do
|
104
|
-
it 'reads without delimiter' do
|
105
|
-
result = IOStreams::Delimited::Reader.open(@file_name) do |io|
|
106
|
-
io.read
|
107
|
-
end
|
108
|
-
file = File.read(@file_name)
|
109
|
-
assert_equal file, result
|
110
|
-
end
|
111
|
-
end
|
112
|
-
|
113
|
-
end
|
114
|
-
end
|
115
|
-
end
|
@@ -1,44 +0,0 @@
|
|
1
|
-
require_relative 'test_helper'
|
2
|
-
|
3
|
-
module Streams
|
4
|
-
class DelimitedWriterTest < Minitest::Test
|
5
|
-
describe IOStreams::Delimited::Writer do
|
6
|
-
before do
|
7
|
-
@file_name = File.join(File.dirname(__FILE__), 'files', 'text.txt')
|
8
|
-
@file = File.read(@file_name)
|
9
|
-
@data = @file.lines
|
10
|
-
end
|
11
|
-
|
12
|
-
describe '#<<' do
|
13
|
-
it 'file' do
|
14
|
-
temp_file = Tempfile.new('rocket_job')
|
15
|
-
file_name = temp_file.to_path
|
16
|
-
IOStreams::Delimited::Writer.open(file_name) do |io|
|
17
|
-
@data.each { |line| io << line.strip }
|
18
|
-
end
|
19
|
-
result = File.read(file_name)
|
20
|
-
assert_equal @file, result
|
21
|
-
end
|
22
|
-
|
23
|
-
it 'stream' do
|
24
|
-
io_string = StringIO.new
|
25
|
-
IOStreams::Delimited::Writer.open(io_string) do |io|
|
26
|
-
@data.each { |line| io << line.strip }
|
27
|
-
end
|
28
|
-
assert_equal @file, io_string.string
|
29
|
-
end
|
30
|
-
end
|
31
|
-
|
32
|
-
describe '.write' do
|
33
|
-
it 'writes without delimiter' do
|
34
|
-
io_string = StringIO.new
|
35
|
-
IOStreams::File::Writer.open(io_string) do |io|
|
36
|
-
io.write(@file)
|
37
|
-
end
|
38
|
-
assert_equal @file, io_string.string
|
39
|
-
end
|
40
|
-
end
|
41
|
-
|
42
|
-
end
|
43
|
-
end
|
44
|
-
end
|