iostreams 0.14.0 → 0.15.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/LICENSE +202 -0
- data/README.md +155 -47
- data/lib/io_streams/file/reader.rb +7 -8
- data/lib/io_streams/file/writer.rb +7 -8
- data/lib/io_streams/io_streams.rb +313 -129
- data/lib/io_streams/{delimited → line}/reader.rb +20 -30
- data/lib/io_streams/line/writer.rb +81 -0
- data/lib/io_streams/pgp.rb +4 -14
- data/lib/io_streams/record/reader.rb +55 -0
- data/lib/io_streams/record/writer.rb +63 -0
- data/lib/io_streams/row/reader.rb +60 -0
- data/lib/io_streams/row/writer.rb +62 -0
- data/lib/io_streams/s3.rb +25 -0
- data/lib/io_streams/s3/reader.rb +64 -0
- data/lib/io_streams/s3/writer.rb +13 -0
- data/lib/io_streams/streams.rb +1 -1
- data/lib/io_streams/tabular.rb +163 -0
- data/lib/io_streams/tabular/errors.rb +14 -0
- data/lib/io_streams/tabular/header.rb +146 -0
- data/lib/io_streams/tabular/parser/array.rb +26 -0
- data/lib/io_streams/tabular/parser/base.rb +12 -0
- data/lib/io_streams/tabular/parser/csv.rb +35 -0
- data/lib/io_streams/tabular/parser/fixed.rb +88 -0
- data/lib/io_streams/tabular/parser/hash.rb +21 -0
- data/lib/io_streams/tabular/parser/json.rb +25 -0
- data/lib/io_streams/tabular/parser/psv.rb +34 -0
- data/lib/io_streams/tabular/utility/csv_row.rb +115 -0
- data/lib/io_streams/version.rb +2 -2
- data/lib/io_streams/xlsx/reader.rb +1 -1
- data/lib/io_streams/zip/reader.rb +1 -1
- data/lib/io_streams/zip/writer.rb +1 -1
- data/lib/iostreams.rb +21 -10
- data/test/bzip2_reader_test.rb +21 -22
- data/test/bzip2_writer_test.rb +38 -32
- data/test/file_reader_test.rb +19 -18
- data/test/file_writer_test.rb +23 -22
- data/test/files/test.json +3 -0
- data/test/gzip_reader_test.rb +21 -22
- data/test/gzip_writer_test.rb +35 -29
- data/test/io_streams_test.rb +137 -61
- data/test/line_reader_test.rb +105 -0
- data/test/line_writer_test.rb +50 -0
- data/test/pgp_reader_test.rb +29 -29
- data/test/pgp_test.rb +149 -195
- data/test/pgp_writer_test.rb +63 -62
- data/test/record_reader_test.rb +61 -0
- data/test/record_writer_test.rb +73 -0
- data/test/row_reader_test.rb +34 -0
- data/test/row_writer_test.rb +51 -0
- data/test/tabular_test.rb +184 -0
- data/test/xlsx_reader_test.rb +13 -17
- data/test/zip_reader_test.rb +21 -22
- data/test/zip_writer_test.rb +40 -36
- metadata +41 -17
- data/lib/io_streams/csv/reader.rb +0 -21
- data/lib/io_streams/csv/writer.rb +0 -20
- data/lib/io_streams/delimited/writer.rb +0 -67
- data/test/csv_reader_test.rb +0 -34
- data/test/csv_writer_test.rb +0 -35
- data/test/delimited_reader_test.rb +0 -115
- data/test/delimited_writer_test.rb +0 -44
data/test/csv_reader_test.rb
DELETED
@@ -1,34 +0,0 @@
|
|
1
|
-
require_relative 'test_helper'
|
2
|
-
|
3
|
-
# Unit Test for IOStreams::File
|
4
|
-
module Streams
|
5
|
-
class CSVReaderTest < Minitest::Test
|
6
|
-
describe IOStreams::CSV::Reader do
|
7
|
-
before do
|
8
|
-
@file_name = File.join(File.dirname(__FILE__), 'files', 'test.csv')
|
9
|
-
@data = CSV.read(@file_name)
|
10
|
-
end
|
11
|
-
|
12
|
-
describe '.open' do
|
13
|
-
it 'file' do
|
14
|
-
rows = []
|
15
|
-
IOStreams::CSV::Reader.open(@file_name) do |io|
|
16
|
-
io.each { |row| rows << row }
|
17
|
-
end
|
18
|
-
assert_equal @data, rows
|
19
|
-
end
|
20
|
-
|
21
|
-
it 'stream' do
|
22
|
-
rows = []
|
23
|
-
File.open(@file_name) do |file|
|
24
|
-
IOStreams::CSV::Reader.open(file) do |io|
|
25
|
-
io.each { |row| rows << row }
|
26
|
-
end
|
27
|
-
end
|
28
|
-
assert_equal @data, rows
|
29
|
-
end
|
30
|
-
end
|
31
|
-
|
32
|
-
end
|
33
|
-
end
|
34
|
-
end
|
data/test/csv_writer_test.rb
DELETED
@@ -1,35 +0,0 @@
|
|
1
|
-
require_relative 'test_helper'
|
2
|
-
require 'csv'
|
3
|
-
|
4
|
-
module Streams
|
5
|
-
class CSVWriterTest < Minitest::Test
|
6
|
-
describe IOStreams::CSV::Writer do
|
7
|
-
before do
|
8
|
-
@file_name = File.join(File.dirname(__FILE__), 'files', 'test.csv')
|
9
|
-
@data = ::CSV.read(@file_name)
|
10
|
-
@raw_csv_data = ::File.read(@file_name)
|
11
|
-
end
|
12
|
-
|
13
|
-
describe '.open' do
|
14
|
-
it 'file' do
|
15
|
-
temp_file = Tempfile.new('rocket_job')
|
16
|
-
file_name = temp_file.to_path
|
17
|
-
IOStreams::CSV::Writer.open(file_name) do |io|
|
18
|
-
@data.each { |row| io << row }
|
19
|
-
end
|
20
|
-
result = File.read(file_name)
|
21
|
-
assert_equal @raw_csv_data, result
|
22
|
-
end
|
23
|
-
|
24
|
-
it 'stream' do
|
25
|
-
io_string = StringIO.new
|
26
|
-
IOStreams::CSV::Writer.open(io_string) do |io|
|
27
|
-
@data.each { |row| io << row }
|
28
|
-
end
|
29
|
-
assert_equal @raw_csv_data, io_string.string
|
30
|
-
end
|
31
|
-
end
|
32
|
-
|
33
|
-
end
|
34
|
-
end
|
35
|
-
end
|
@@ -1,115 +0,0 @@
|
|
1
|
-
require_relative 'test_helper'
|
2
|
-
|
3
|
-
# Unit Test for IOStreams::File
|
4
|
-
module Streams
|
5
|
-
class DelimitedReaderTest < Minitest::Test
|
6
|
-
describe IOStreams::Delimited::Reader do
|
7
|
-
before do
|
8
|
-
@file_name = File.join(File.dirname(__FILE__), 'files', 'text.txt')
|
9
|
-
@data = []
|
10
|
-
File.open(@file_name, 'rt') do |file|
|
11
|
-
while !file.eof?
|
12
|
-
@data << file.readline.strip
|
13
|
-
end
|
14
|
-
end
|
15
|
-
end
|
16
|
-
|
17
|
-
describe '#initialize' do
|
18
|
-
it 'does not strip invalid characters' do
|
19
|
-
bad_lines = [
|
20
|
-
"New M\xE9xico,NE",
|
21
|
-
'good line',
|
22
|
-
"New M\xE9xico,SF"
|
23
|
-
]
|
24
|
-
input = StringIO.new(bad_lines.join("\n"))
|
25
|
-
lines = []
|
26
|
-
IOStreams::Delimited::Reader.open(input) do |io|
|
27
|
-
assert_equal false, io.strip_non_printable
|
28
|
-
assert_raises ArgumentError do
|
29
|
-
io.each { |line| lines << line }
|
30
|
-
end
|
31
|
-
end
|
32
|
-
end
|
33
|
-
|
34
|
-
it 'strips invalid characters' do
|
35
|
-
bad_lines = [
|
36
|
-
"New M\xE9xico,NE",
|
37
|
-
'good line',
|
38
|
-
"New M\xE9xico,SF"
|
39
|
-
]
|
40
|
-
fixed_lines = bad_lines.collect { |line| line.force_encoding('BINARY').gsub(/[^[:print:]|\r|\n]/, '') }
|
41
|
-
input = StringIO.new(bad_lines.join("\n"))
|
42
|
-
lines = []
|
43
|
-
IOStreams::Delimited::Reader.open(input, strip_non_printable: true) do |io|
|
44
|
-
assert_equal true, io.strip_non_printable
|
45
|
-
io.each { |line| lines << line }
|
46
|
-
end
|
47
|
-
assert_equal fixed_lines, lines
|
48
|
-
end
|
49
|
-
end
|
50
|
-
|
51
|
-
describe '#each' do
|
52
|
-
it 'each_line file' do
|
53
|
-
lines = []
|
54
|
-
IOStreams::Delimited::Reader.open(@file_name) do |io|
|
55
|
-
io.each { |line| lines << line }
|
56
|
-
end
|
57
|
-
assert_equal @data, lines
|
58
|
-
end
|
59
|
-
|
60
|
-
it 'each_line stream' do
|
61
|
-
lines = []
|
62
|
-
File.open(@file_name) do |file|
|
63
|
-
IOStreams::Delimited::Reader.open(file) do |io|
|
64
|
-
io.each { |line| lines << line }
|
65
|
-
end
|
66
|
-
end
|
67
|
-
assert_equal @data, lines
|
68
|
-
end
|
69
|
-
|
70
|
-
["\r\n", "\n\r", "\n", "\r"].each do |delimiter|
|
71
|
-
it "autodetect delimiter: #{delimiter.inspect}" do
|
72
|
-
lines = []
|
73
|
-
stream = StringIO.new(@data.join(delimiter))
|
74
|
-
IOStreams::Delimited::Reader.open(stream, buffer_size: 15) do |io|
|
75
|
-
io.each { |line| lines << line }
|
76
|
-
end
|
77
|
-
assert_equal @data, lines
|
78
|
-
end
|
79
|
-
end
|
80
|
-
|
81
|
-
['@', 'BLAH'].each do |delimiter|
|
82
|
-
it "reads delimited #{delimiter.inspect}" do
|
83
|
-
lines = []
|
84
|
-
stream = StringIO.new(@data.join(delimiter))
|
85
|
-
IOStreams::Delimited::Reader.open(stream, buffer_size: 15, delimiter: delimiter) do |io|
|
86
|
-
io.each { |line| lines << line }
|
87
|
-
end
|
88
|
-
assert_equal @data, lines
|
89
|
-
end
|
90
|
-
end
|
91
|
-
|
92
|
-
it 'reads binary delimited' do
|
93
|
-
delimiter = "\x01"
|
94
|
-
lines = []
|
95
|
-
stream = StringIO.new(@data.join(delimiter))
|
96
|
-
IOStreams::Delimited::Reader.open(stream, buffer_size: 15, delimiter: delimiter, encoding: IOStreams::BINARY_ENCODING) do |io|
|
97
|
-
io.each { |line| lines << line }
|
98
|
-
end
|
99
|
-
assert_equal @data, lines
|
100
|
-
end
|
101
|
-
end
|
102
|
-
|
103
|
-
describe '.read' do
|
104
|
-
it 'reads without delimiter' do
|
105
|
-
result = IOStreams::Delimited::Reader.open(@file_name) do |io|
|
106
|
-
io.read
|
107
|
-
end
|
108
|
-
file = File.read(@file_name)
|
109
|
-
assert_equal file, result
|
110
|
-
end
|
111
|
-
end
|
112
|
-
|
113
|
-
end
|
114
|
-
end
|
115
|
-
end
|
@@ -1,44 +0,0 @@
|
|
1
|
-
require_relative 'test_helper'
|
2
|
-
|
3
|
-
module Streams
|
4
|
-
class DelimitedWriterTest < Minitest::Test
|
5
|
-
describe IOStreams::Delimited::Writer do
|
6
|
-
before do
|
7
|
-
@file_name = File.join(File.dirname(__FILE__), 'files', 'text.txt')
|
8
|
-
@file = File.read(@file_name)
|
9
|
-
@data = @file.lines
|
10
|
-
end
|
11
|
-
|
12
|
-
describe '#<<' do
|
13
|
-
it 'file' do
|
14
|
-
temp_file = Tempfile.new('rocket_job')
|
15
|
-
file_name = temp_file.to_path
|
16
|
-
IOStreams::Delimited::Writer.open(file_name) do |io|
|
17
|
-
@data.each { |line| io << line.strip }
|
18
|
-
end
|
19
|
-
result = File.read(file_name)
|
20
|
-
assert_equal @file, result
|
21
|
-
end
|
22
|
-
|
23
|
-
it 'stream' do
|
24
|
-
io_string = StringIO.new
|
25
|
-
IOStreams::Delimited::Writer.open(io_string) do |io|
|
26
|
-
@data.each { |line| io << line.strip }
|
27
|
-
end
|
28
|
-
assert_equal @file, io_string.string
|
29
|
-
end
|
30
|
-
end
|
31
|
-
|
32
|
-
describe '.write' do
|
33
|
-
it 'writes without delimiter' do
|
34
|
-
io_string = StringIO.new
|
35
|
-
IOStreams::File::Writer.open(io_string) do |io|
|
36
|
-
io.write(@file)
|
37
|
-
end
|
38
|
-
assert_equal @file, io_string.string
|
39
|
-
end
|
40
|
-
end
|
41
|
-
|
42
|
-
end
|
43
|
-
end
|
44
|
-
end
|