iostreams 0.14.0 → 0.15.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (62) hide show
  1. checksums.yaml +4 -4
  2. data/LICENSE +202 -0
  3. data/README.md +155 -47
  4. data/lib/io_streams/file/reader.rb +7 -8
  5. data/lib/io_streams/file/writer.rb +7 -8
  6. data/lib/io_streams/io_streams.rb +313 -129
  7. data/lib/io_streams/{delimited → line}/reader.rb +20 -30
  8. data/lib/io_streams/line/writer.rb +81 -0
  9. data/lib/io_streams/pgp.rb +4 -14
  10. data/lib/io_streams/record/reader.rb +55 -0
  11. data/lib/io_streams/record/writer.rb +63 -0
  12. data/lib/io_streams/row/reader.rb +60 -0
  13. data/lib/io_streams/row/writer.rb +62 -0
  14. data/lib/io_streams/s3.rb +25 -0
  15. data/lib/io_streams/s3/reader.rb +64 -0
  16. data/lib/io_streams/s3/writer.rb +13 -0
  17. data/lib/io_streams/streams.rb +1 -1
  18. data/lib/io_streams/tabular.rb +163 -0
  19. data/lib/io_streams/tabular/errors.rb +14 -0
  20. data/lib/io_streams/tabular/header.rb +146 -0
  21. data/lib/io_streams/tabular/parser/array.rb +26 -0
  22. data/lib/io_streams/tabular/parser/base.rb +12 -0
  23. data/lib/io_streams/tabular/parser/csv.rb +35 -0
  24. data/lib/io_streams/tabular/parser/fixed.rb +88 -0
  25. data/lib/io_streams/tabular/parser/hash.rb +21 -0
  26. data/lib/io_streams/tabular/parser/json.rb +25 -0
  27. data/lib/io_streams/tabular/parser/psv.rb +34 -0
  28. data/lib/io_streams/tabular/utility/csv_row.rb +115 -0
  29. data/lib/io_streams/version.rb +2 -2
  30. data/lib/io_streams/xlsx/reader.rb +1 -1
  31. data/lib/io_streams/zip/reader.rb +1 -1
  32. data/lib/io_streams/zip/writer.rb +1 -1
  33. data/lib/iostreams.rb +21 -10
  34. data/test/bzip2_reader_test.rb +21 -22
  35. data/test/bzip2_writer_test.rb +38 -32
  36. data/test/file_reader_test.rb +19 -18
  37. data/test/file_writer_test.rb +23 -22
  38. data/test/files/test.json +3 -0
  39. data/test/gzip_reader_test.rb +21 -22
  40. data/test/gzip_writer_test.rb +35 -29
  41. data/test/io_streams_test.rb +137 -61
  42. data/test/line_reader_test.rb +105 -0
  43. data/test/line_writer_test.rb +50 -0
  44. data/test/pgp_reader_test.rb +29 -29
  45. data/test/pgp_test.rb +149 -195
  46. data/test/pgp_writer_test.rb +63 -62
  47. data/test/record_reader_test.rb +61 -0
  48. data/test/record_writer_test.rb +73 -0
  49. data/test/row_reader_test.rb +34 -0
  50. data/test/row_writer_test.rb +51 -0
  51. data/test/tabular_test.rb +184 -0
  52. data/test/xlsx_reader_test.rb +13 -17
  53. data/test/zip_reader_test.rb +21 -22
  54. data/test/zip_writer_test.rb +40 -36
  55. metadata +41 -17
  56. data/lib/io_streams/csv/reader.rb +0 -21
  57. data/lib/io_streams/csv/writer.rb +0 -20
  58. data/lib/io_streams/delimited/writer.rb +0 -67
  59. data/test/csv_reader_test.rb +0 -34
  60. data/test/csv_writer_test.rb +0 -35
  61. data/test/delimited_reader_test.rb +0 -115
  62. data/test/delimited_writer_test.rb +0 -44
@@ -1,34 +0,0 @@
1
- require_relative 'test_helper'
2
-
3
- # Unit Test for IOStreams::File
4
- module Streams
5
- class CSVReaderTest < Minitest::Test
6
- describe IOStreams::CSV::Reader do
7
- before do
8
- @file_name = File.join(File.dirname(__FILE__), 'files', 'test.csv')
9
- @data = CSV.read(@file_name)
10
- end
11
-
12
- describe '.open' do
13
- it 'file' do
14
- rows = []
15
- IOStreams::CSV::Reader.open(@file_name) do |io|
16
- io.each { |row| rows << row }
17
- end
18
- assert_equal @data, rows
19
- end
20
-
21
- it 'stream' do
22
- rows = []
23
- File.open(@file_name) do |file|
24
- IOStreams::CSV::Reader.open(file) do |io|
25
- io.each { |row| rows << row }
26
- end
27
- end
28
- assert_equal @data, rows
29
- end
30
- end
31
-
32
- end
33
- end
34
- end
@@ -1,35 +0,0 @@
1
- require_relative 'test_helper'
2
- require 'csv'
3
-
4
- module Streams
5
- class CSVWriterTest < Minitest::Test
6
- describe IOStreams::CSV::Writer do
7
- before do
8
- @file_name = File.join(File.dirname(__FILE__), 'files', 'test.csv')
9
- @data = ::CSV.read(@file_name)
10
- @raw_csv_data = ::File.read(@file_name)
11
- end
12
-
13
- describe '.open' do
14
- it 'file' do
15
- temp_file = Tempfile.new('rocket_job')
16
- file_name = temp_file.to_path
17
- IOStreams::CSV::Writer.open(file_name) do |io|
18
- @data.each { |row| io << row }
19
- end
20
- result = File.read(file_name)
21
- assert_equal @raw_csv_data, result
22
- end
23
-
24
- it 'stream' do
25
- io_string = StringIO.new
26
- IOStreams::CSV::Writer.open(io_string) do |io|
27
- @data.each { |row| io << row }
28
- end
29
- assert_equal @raw_csv_data, io_string.string
30
- end
31
- end
32
-
33
- end
34
- end
35
- end
@@ -1,115 +0,0 @@
1
- require_relative 'test_helper'
2
-
3
- # Unit Test for IOStreams::File
4
- module Streams
5
- class DelimitedReaderTest < Minitest::Test
6
- describe IOStreams::Delimited::Reader do
7
- before do
8
- @file_name = File.join(File.dirname(__FILE__), 'files', 'text.txt')
9
- @data = []
10
- File.open(@file_name, 'rt') do |file|
11
- while !file.eof?
12
- @data << file.readline.strip
13
- end
14
- end
15
- end
16
-
17
- describe '#initialize' do
18
- it 'does not strip invalid characters' do
19
- bad_lines = [
20
- "New M\xE9xico,NE",
21
- 'good line',
22
- "New M\xE9xico,SF"
23
- ]
24
- input = StringIO.new(bad_lines.join("\n"))
25
- lines = []
26
- IOStreams::Delimited::Reader.open(input) do |io|
27
- assert_equal false, io.strip_non_printable
28
- assert_raises ArgumentError do
29
- io.each { |line| lines << line }
30
- end
31
- end
32
- end
33
-
34
- it 'strips invalid characters' do
35
- bad_lines = [
36
- "New M\xE9xico,NE",
37
- 'good line',
38
- "New M\xE9xico,SF"
39
- ]
40
- fixed_lines = bad_lines.collect { |line| line.force_encoding('BINARY').gsub(/[^[:print:]|\r|\n]/, '') }
41
- input = StringIO.new(bad_lines.join("\n"))
42
- lines = []
43
- IOStreams::Delimited::Reader.open(input, strip_non_printable: true) do |io|
44
- assert_equal true, io.strip_non_printable
45
- io.each { |line| lines << line }
46
- end
47
- assert_equal fixed_lines, lines
48
- end
49
- end
50
-
51
- describe '#each' do
52
- it 'each_line file' do
53
- lines = []
54
- IOStreams::Delimited::Reader.open(@file_name) do |io|
55
- io.each { |line| lines << line }
56
- end
57
- assert_equal @data, lines
58
- end
59
-
60
- it 'each_line stream' do
61
- lines = []
62
- File.open(@file_name) do |file|
63
- IOStreams::Delimited::Reader.open(file) do |io|
64
- io.each { |line| lines << line }
65
- end
66
- end
67
- assert_equal @data, lines
68
- end
69
-
70
- ["\r\n", "\n\r", "\n", "\r"].each do |delimiter|
71
- it "autodetect delimiter: #{delimiter.inspect}" do
72
- lines = []
73
- stream = StringIO.new(@data.join(delimiter))
74
- IOStreams::Delimited::Reader.open(stream, buffer_size: 15) do |io|
75
- io.each { |line| lines << line }
76
- end
77
- assert_equal @data, lines
78
- end
79
- end
80
-
81
- ['@', 'BLAH'].each do |delimiter|
82
- it "reads delimited #{delimiter.inspect}" do
83
- lines = []
84
- stream = StringIO.new(@data.join(delimiter))
85
- IOStreams::Delimited::Reader.open(stream, buffer_size: 15, delimiter: delimiter) do |io|
86
- io.each { |line| lines << line }
87
- end
88
- assert_equal @data, lines
89
- end
90
- end
91
-
92
- it 'reads binary delimited' do
93
- delimiter = "\x01"
94
- lines = []
95
- stream = StringIO.new(@data.join(delimiter))
96
- IOStreams::Delimited::Reader.open(stream, buffer_size: 15, delimiter: delimiter, encoding: IOStreams::BINARY_ENCODING) do |io|
97
- io.each { |line| lines << line }
98
- end
99
- assert_equal @data, lines
100
- end
101
- end
102
-
103
- describe '.read' do
104
- it 'reads without delimiter' do
105
- result = IOStreams::Delimited::Reader.open(@file_name) do |io|
106
- io.read
107
- end
108
- file = File.read(@file_name)
109
- assert_equal file, result
110
- end
111
- end
112
-
113
- end
114
- end
115
- end
@@ -1,44 +0,0 @@
1
- require_relative 'test_helper'
2
-
3
- module Streams
4
- class DelimitedWriterTest < Minitest::Test
5
- describe IOStreams::Delimited::Writer do
6
- before do
7
- @file_name = File.join(File.dirname(__FILE__), 'files', 'text.txt')
8
- @file = File.read(@file_name)
9
- @data = @file.lines
10
- end
11
-
12
- describe '#<<' do
13
- it 'file' do
14
- temp_file = Tempfile.new('rocket_job')
15
- file_name = temp_file.to_path
16
- IOStreams::Delimited::Writer.open(file_name) do |io|
17
- @data.each { |line| io << line.strip }
18
- end
19
- result = File.read(file_name)
20
- assert_equal @file, result
21
- end
22
-
23
- it 'stream' do
24
- io_string = StringIO.new
25
- IOStreams::Delimited::Writer.open(io_string) do |io|
26
- @data.each { |line| io << line.strip }
27
- end
28
- assert_equal @file, io_string.string
29
- end
30
- end
31
-
32
- describe '.write' do
33
- it 'writes without delimiter' do
34
- io_string = StringIO.new
35
- IOStreams::File::Writer.open(io_string) do |io|
36
- io.write(@file)
37
- end
38
- assert_equal @file, io_string.string
39
- end
40
- end
41
-
42
- end
43
- end
44
- end