iostreams 0.14.0 → 0.15.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (62) hide show
  1. checksums.yaml +4 -4
  2. data/LICENSE +202 -0
  3. data/README.md +155 -47
  4. data/lib/io_streams/file/reader.rb +7 -8
  5. data/lib/io_streams/file/writer.rb +7 -8
  6. data/lib/io_streams/io_streams.rb +313 -129
  7. data/lib/io_streams/{delimited → line}/reader.rb +20 -30
  8. data/lib/io_streams/line/writer.rb +81 -0
  9. data/lib/io_streams/pgp.rb +4 -14
  10. data/lib/io_streams/record/reader.rb +55 -0
  11. data/lib/io_streams/record/writer.rb +63 -0
  12. data/lib/io_streams/row/reader.rb +60 -0
  13. data/lib/io_streams/row/writer.rb +62 -0
  14. data/lib/io_streams/s3.rb +25 -0
  15. data/lib/io_streams/s3/reader.rb +64 -0
  16. data/lib/io_streams/s3/writer.rb +13 -0
  17. data/lib/io_streams/streams.rb +1 -1
  18. data/lib/io_streams/tabular.rb +163 -0
  19. data/lib/io_streams/tabular/errors.rb +14 -0
  20. data/lib/io_streams/tabular/header.rb +146 -0
  21. data/lib/io_streams/tabular/parser/array.rb +26 -0
  22. data/lib/io_streams/tabular/parser/base.rb +12 -0
  23. data/lib/io_streams/tabular/parser/csv.rb +35 -0
  24. data/lib/io_streams/tabular/parser/fixed.rb +88 -0
  25. data/lib/io_streams/tabular/parser/hash.rb +21 -0
  26. data/lib/io_streams/tabular/parser/json.rb +25 -0
  27. data/lib/io_streams/tabular/parser/psv.rb +34 -0
  28. data/lib/io_streams/tabular/utility/csv_row.rb +115 -0
  29. data/lib/io_streams/version.rb +2 -2
  30. data/lib/io_streams/xlsx/reader.rb +1 -1
  31. data/lib/io_streams/zip/reader.rb +1 -1
  32. data/lib/io_streams/zip/writer.rb +1 -1
  33. data/lib/iostreams.rb +21 -10
  34. data/test/bzip2_reader_test.rb +21 -22
  35. data/test/bzip2_writer_test.rb +38 -32
  36. data/test/file_reader_test.rb +19 -18
  37. data/test/file_writer_test.rb +23 -22
  38. data/test/files/test.json +3 -0
  39. data/test/gzip_reader_test.rb +21 -22
  40. data/test/gzip_writer_test.rb +35 -29
  41. data/test/io_streams_test.rb +137 -61
  42. data/test/line_reader_test.rb +105 -0
  43. data/test/line_writer_test.rb +50 -0
  44. data/test/pgp_reader_test.rb +29 -29
  45. data/test/pgp_test.rb +149 -195
  46. data/test/pgp_writer_test.rb +63 -62
  47. data/test/record_reader_test.rb +61 -0
  48. data/test/record_writer_test.rb +73 -0
  49. data/test/row_reader_test.rb +34 -0
  50. data/test/row_writer_test.rb +51 -0
  51. data/test/tabular_test.rb +184 -0
  52. data/test/xlsx_reader_test.rb +13 -17
  53. data/test/zip_reader_test.rb +21 -22
  54. data/test/zip_writer_test.rb +40 -36
  55. metadata +41 -17
  56. data/lib/io_streams/csv/reader.rb +0 -21
  57. data/lib/io_streams/csv/writer.rb +0 -20
  58. data/lib/io_streams/delimited/writer.rb +0 -67
  59. data/test/csv_reader_test.rb +0 -34
  60. data/test/csv_writer_test.rb +0 -35
  61. data/test/delimited_reader_test.rb +0 -115
  62. data/test/delimited_writer_test.rb +0 -44
@@ -1,34 +0,0 @@
1
- require_relative 'test_helper'
2
-
3
- # Unit Test for IOStreams::File
4
- module Streams
5
- class CSVReaderTest < Minitest::Test
6
- describe IOStreams::CSV::Reader do
7
- before do
8
- @file_name = File.join(File.dirname(__FILE__), 'files', 'test.csv')
9
- @data = CSV.read(@file_name)
10
- end
11
-
12
- describe '.open' do
13
- it 'file' do
14
- rows = []
15
- IOStreams::CSV::Reader.open(@file_name) do |io|
16
- io.each { |row| rows << row }
17
- end
18
- assert_equal @data, rows
19
- end
20
-
21
- it 'stream' do
22
- rows = []
23
- File.open(@file_name) do |file|
24
- IOStreams::CSV::Reader.open(file) do |io|
25
- io.each { |row| rows << row }
26
- end
27
- end
28
- assert_equal @data, rows
29
- end
30
- end
31
-
32
- end
33
- end
34
- end
@@ -1,35 +0,0 @@
1
- require_relative 'test_helper'
2
- require 'csv'
3
-
4
- module Streams
5
- class CSVWriterTest < Minitest::Test
6
- describe IOStreams::CSV::Writer do
7
- before do
8
- @file_name = File.join(File.dirname(__FILE__), 'files', 'test.csv')
9
- @data = ::CSV.read(@file_name)
10
- @raw_csv_data = ::File.read(@file_name)
11
- end
12
-
13
- describe '.open' do
14
- it 'file' do
15
- temp_file = Tempfile.new('rocket_job')
16
- file_name = temp_file.to_path
17
- IOStreams::CSV::Writer.open(file_name) do |io|
18
- @data.each { |row| io << row }
19
- end
20
- result = File.read(file_name)
21
- assert_equal @raw_csv_data, result
22
- end
23
-
24
- it 'stream' do
25
- io_string = StringIO.new
26
- IOStreams::CSV::Writer.open(io_string) do |io|
27
- @data.each { |row| io << row }
28
- end
29
- assert_equal @raw_csv_data, io_string.string
30
- end
31
- end
32
-
33
- end
34
- end
35
- end
@@ -1,115 +0,0 @@
1
- require_relative 'test_helper'
2
-
3
- # Unit Test for IOStreams::File
4
- module Streams
5
- class DelimitedReaderTest < Minitest::Test
6
- describe IOStreams::Delimited::Reader do
7
- before do
8
- @file_name = File.join(File.dirname(__FILE__), 'files', 'text.txt')
9
- @data = []
10
- File.open(@file_name, 'rt') do |file|
11
- while !file.eof?
12
- @data << file.readline.strip
13
- end
14
- end
15
- end
16
-
17
- describe '#initialize' do
18
- it 'does not strip invalid characters' do
19
- bad_lines = [
20
- "New M\xE9xico,NE",
21
- 'good line',
22
- "New M\xE9xico,SF"
23
- ]
24
- input = StringIO.new(bad_lines.join("\n"))
25
- lines = []
26
- IOStreams::Delimited::Reader.open(input) do |io|
27
- assert_equal false, io.strip_non_printable
28
- assert_raises ArgumentError do
29
- io.each { |line| lines << line }
30
- end
31
- end
32
- end
33
-
34
- it 'strips invalid characters' do
35
- bad_lines = [
36
- "New M\xE9xico,NE",
37
- 'good line',
38
- "New M\xE9xico,SF"
39
- ]
40
- fixed_lines = bad_lines.collect { |line| line.force_encoding('BINARY').gsub(/[^[:print:]|\r|\n]/, '') }
41
- input = StringIO.new(bad_lines.join("\n"))
42
- lines = []
43
- IOStreams::Delimited::Reader.open(input, strip_non_printable: true) do |io|
44
- assert_equal true, io.strip_non_printable
45
- io.each { |line| lines << line }
46
- end
47
- assert_equal fixed_lines, lines
48
- end
49
- end
50
-
51
- describe '#each' do
52
- it 'each_line file' do
53
- lines = []
54
- IOStreams::Delimited::Reader.open(@file_name) do |io|
55
- io.each { |line| lines << line }
56
- end
57
- assert_equal @data, lines
58
- end
59
-
60
- it 'each_line stream' do
61
- lines = []
62
- File.open(@file_name) do |file|
63
- IOStreams::Delimited::Reader.open(file) do |io|
64
- io.each { |line| lines << line }
65
- end
66
- end
67
- assert_equal @data, lines
68
- end
69
-
70
- ["\r\n", "\n\r", "\n", "\r"].each do |delimiter|
71
- it "autodetect delimiter: #{delimiter.inspect}" do
72
- lines = []
73
- stream = StringIO.new(@data.join(delimiter))
74
- IOStreams::Delimited::Reader.open(stream, buffer_size: 15) do |io|
75
- io.each { |line| lines << line }
76
- end
77
- assert_equal @data, lines
78
- end
79
- end
80
-
81
- ['@', 'BLAH'].each do |delimiter|
82
- it "reads delimited #{delimiter.inspect}" do
83
- lines = []
84
- stream = StringIO.new(@data.join(delimiter))
85
- IOStreams::Delimited::Reader.open(stream, buffer_size: 15, delimiter: delimiter) do |io|
86
- io.each { |line| lines << line }
87
- end
88
- assert_equal @data, lines
89
- end
90
- end
91
-
92
- it 'reads binary delimited' do
93
- delimiter = "\x01"
94
- lines = []
95
- stream = StringIO.new(@data.join(delimiter))
96
- IOStreams::Delimited::Reader.open(stream, buffer_size: 15, delimiter: delimiter, encoding: IOStreams::BINARY_ENCODING) do |io|
97
- io.each { |line| lines << line }
98
- end
99
- assert_equal @data, lines
100
- end
101
- end
102
-
103
- describe '.read' do
104
- it 'reads without delimiter' do
105
- result = IOStreams::Delimited::Reader.open(@file_name) do |io|
106
- io.read
107
- end
108
- file = File.read(@file_name)
109
- assert_equal file, result
110
- end
111
- end
112
-
113
- end
114
- end
115
- end
@@ -1,44 +0,0 @@
1
- require_relative 'test_helper'
2
-
3
- module Streams
4
- class DelimitedWriterTest < Minitest::Test
5
- describe IOStreams::Delimited::Writer do
6
- before do
7
- @file_name = File.join(File.dirname(__FILE__), 'files', 'text.txt')
8
- @file = File.read(@file_name)
9
- @data = @file.lines
10
- end
11
-
12
- describe '#<<' do
13
- it 'file' do
14
- temp_file = Tempfile.new('rocket_job')
15
- file_name = temp_file.to_path
16
- IOStreams::Delimited::Writer.open(file_name) do |io|
17
- @data.each { |line| io << line.strip }
18
- end
19
- result = File.read(file_name)
20
- assert_equal @file, result
21
- end
22
-
23
- it 'stream' do
24
- io_string = StringIO.new
25
- IOStreams::Delimited::Writer.open(io_string) do |io|
26
- @data.each { |line| io << line.strip }
27
- end
28
- assert_equal @file, io_string.string
29
- end
30
- end
31
-
32
- describe '.write' do
33
- it 'writes without delimiter' do
34
- io_string = StringIO.new
35
- IOStreams::File::Writer.open(io_string) do |io|
36
- io.write(@file)
37
- end
38
- assert_equal @file, io_string.string
39
- end
40
- end
41
-
42
- end
43
- end
44
- end