iostreams 0.9.0 → 0.9.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/io_streams/delimited/reader.rb +2 -3
- data/lib/io_streams/io_streams.rb +1 -1
- data/lib/io_streams/version.rb +1 -1
- data/lib/io_streams/zip/reader.rb +9 -7
- data/lib/io_streams/zip/writer.rb +9 -8
- data/test/delimited_reader_test.rb +35 -1
- metadata +3 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 4c229181098f8b4aedf3ce542477dc44554f94f4
|
4
|
+
data.tar.gz: 866c35496433dbd255865b322a61bd50272b5ea5
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: d91a975d92b3ab08d2d0185a927f6403ccdd362b63dd07871f7322219d07ea088e84ff401c9bd6ec3cb8227e29a7d7702aec6d35dbe863a2764ebda83e5e3dc9
|
7
|
+
data.tar.gz: 7204cf47a99466b5818b462f3cffbc303f129e277793c3fda1ec8ef1c5f697145bcda34d69ea859d674789cb3f500afcd29fcacad76963c173bdd9768757f057
|
@@ -1,7 +1,7 @@
|
|
1
1
|
module IOStreams
|
2
2
|
module Delimited
|
3
3
|
class Reader
|
4
|
-
attr_accessor :delimiter
|
4
|
+
attr_accessor :delimiter, :buffer_size, :encoding, :strip_non_printable
|
5
5
|
|
6
6
|
# Read from a file or stream
|
7
7
|
def self.open(file_name_or_io, options={}, &block)
|
@@ -54,8 +54,7 @@ module IOStreams
|
|
54
54
|
@delimiter = options.delete(:delimiter)
|
55
55
|
@buffer_size = options.delete(:buffer_size) || 65536
|
56
56
|
@encoding = options.has_key?(:encoding) ? options.delete(:encoding) : UTF8_ENCODING
|
57
|
-
@strip_non_printable = options.delete(:strip_non_printable)
|
58
|
-
@strip_non_printable = @strip_non_printable.nil? && (@encoding == UTF8_ENCODING)
|
57
|
+
@strip_non_printable = options.delete(:strip_non_printable) || false
|
59
58
|
raise ArgumentError.new("Unknown IOStreams::Delimited::Reader#initialize options: #{options.inspect}") if options.size > 0
|
60
59
|
|
61
60
|
@delimiter.force_encoding(UTF8_ENCODING) if @delimiter && @encoding
|
@@ -32,7 +32,7 @@ module IOStreams
|
|
32
32
|
# => [ :file ]
|
33
33
|
def self.streams_for_file_name(file_name)
|
34
34
|
raise ArgumentError.new('File name cannot be nil') if file_name.nil?
|
35
|
-
raise ArgumentError.new("
|
35
|
+
raise ArgumentError.new("File name must be a string: #{file_name.inspect}, class: #{file_name.class}") unless file_name.is_a?(String)
|
36
36
|
parts = file_name.split('.')
|
37
37
|
extensions = []
|
38
38
|
while extension = parts.pop
|
data/lib/io_streams/version.rb
CHANGED
@@ -17,6 +17,15 @@ module IOStreams
|
|
17
17
|
buffer_size = options.delete(:buffer_size) || 65536
|
18
18
|
raise(ArgumentError, "Unknown IOStreams::Zip::Reader option: #{options.inspect}") if options.size > 0
|
19
19
|
|
20
|
+
if !defined?(JRuby) && !defined?(::Zip)
|
21
|
+
# MRI needs Ruby Zip, since it only has native support for GZip
|
22
|
+
begin
|
23
|
+
require 'zip'
|
24
|
+
rescue LoadError => exc
|
25
|
+
raise(LoadError, "Install gem 'rubyzip' to read and write Zip files: #{exc.message}")
|
26
|
+
end
|
27
|
+
end
|
28
|
+
|
20
29
|
# File name supplied
|
21
30
|
return read_file(file_name_or_io, &block) unless IOStreams.reader_stream?(file_name_or_io)
|
22
31
|
|
@@ -50,13 +59,6 @@ module IOStreams
|
|
50
59
|
end
|
51
60
|
|
52
61
|
else
|
53
|
-
# MRI needs Ruby Zip, since it only has native support for GZip
|
54
|
-
begin
|
55
|
-
require 'zip'
|
56
|
-
rescue LoadError => exc
|
57
|
-
puts 'Please install gem rubyzip so that RocketJob can read Zip files in Ruby MRI'
|
58
|
-
raise(exc)
|
59
|
-
end
|
60
62
|
|
61
63
|
# Read from a zip file or stream, decompressing the contents as it is read
|
62
64
|
# The input stream from the first file found in the zip file is passed
|
@@ -32,6 +32,15 @@ module IOStreams
|
|
32
32
|
zip_file_name = file_name_or_io.to_s[0..-5] if zip_file_name.nil? && !file_name_or_io.respond_to?(:write) && (file_name_or_io =~ /\.(zip)\z/)
|
33
33
|
zip_file_name ||= 'file'
|
34
34
|
|
35
|
+
if !defined?(JRuby) && !defined?(::Zip)
|
36
|
+
# MRI needs Ruby Zip, since it only has native support for GZip
|
37
|
+
begin
|
38
|
+
require 'zip'
|
39
|
+
rescue LoadError => exc
|
40
|
+
raise(LoadError, "Install gem 'rubyzip' to read and write Zip files: #{exc.message}")
|
41
|
+
end
|
42
|
+
end
|
43
|
+
|
35
44
|
# File name supplied
|
36
45
|
return write_file(file_name_or_io, zip_file_name, &block) unless IOStreams.writer_stream?(file_name_or_io)
|
37
46
|
|
@@ -64,14 +73,6 @@ module IOStreams
|
|
64
73
|
end
|
65
74
|
|
66
75
|
else
|
67
|
-
# MRI needs Ruby Zip, since it only has native support for GZip
|
68
|
-
begin
|
69
|
-
require 'zip'
|
70
|
-
rescue LoadError => exc
|
71
|
-
puts 'Please install gem rubyzip so that RocketJob can read Zip files in Ruby MRI'
|
72
|
-
raise(exc)
|
73
|
-
end
|
74
|
-
|
75
76
|
def self.write_file(file_name, zip_file_name, &block)
|
76
77
|
zos = ::Zip::OutputStream.new(file_name)
|
77
78
|
zos.put_next_entry(zip_file_name)
|
@@ -14,6 +14,40 @@ module Streams
|
|
14
14
|
end
|
15
15
|
end
|
16
16
|
|
17
|
+
describe '#initialize' do
|
18
|
+
it 'does not strip invalid characters' do
|
19
|
+
bad_lines = [
|
20
|
+
"New M\xE9xico,NE",
|
21
|
+
'good line',
|
22
|
+
"New M\xE9xico,SF"
|
23
|
+
]
|
24
|
+
input = StringIO.new(bad_lines.join("\n"))
|
25
|
+
lines = []
|
26
|
+
IOStreams::Delimited::Reader.open(input) do |io|
|
27
|
+
assert_equal false, io.strip_non_printable
|
28
|
+
assert_raises ArgumentError do
|
29
|
+
io.each { |line| lines << line }
|
30
|
+
end
|
31
|
+
end
|
32
|
+
end
|
33
|
+
|
34
|
+
it 'strips invalid characters' do
|
35
|
+
bad_lines = [
|
36
|
+
"New M\xE9xico,NE",
|
37
|
+
'good line',
|
38
|
+
"New M\xE9xico,SF"
|
39
|
+
]
|
40
|
+
fixed_lines = bad_lines.collect { |line| line.force_encoding('BINARY').gsub(/[^[:print:]|\r|\n]/, '') }
|
41
|
+
input = StringIO.new(bad_lines.join("\n"))
|
42
|
+
lines = []
|
43
|
+
IOStreams::Delimited::Reader.open(input, strip_non_printable: true) do |io|
|
44
|
+
assert_equal true, io.strip_non_printable
|
45
|
+
io.each { |line| lines << line }
|
46
|
+
end
|
47
|
+
assert_equal fixed_lines, lines
|
48
|
+
end
|
49
|
+
end
|
50
|
+
|
17
51
|
describe '#each' do
|
18
52
|
it 'each_line file' do
|
19
53
|
lines = []
|
@@ -71,7 +105,7 @@ module Streams
|
|
71
105
|
result = IOStreams::Delimited::Reader.open(@file_name) do |io|
|
72
106
|
io.read
|
73
107
|
end
|
74
|
-
file
|
108
|
+
file = File.read(@file_name)
|
75
109
|
assert_equal file, result
|
76
110
|
end
|
77
111
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: iostreams
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.9.
|
4
|
+
version: 0.9.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Reid Morrison
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2016-
|
11
|
+
date: 2016-02-20 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: symmetric-encryption
|
@@ -82,7 +82,7 @@ files:
|
|
82
82
|
- test/zip_writer_test.rb
|
83
83
|
homepage: https://github.com/rocketjob/streams
|
84
84
|
licenses:
|
85
|
-
- Apache
|
85
|
+
- Apache-2.0
|
86
86
|
metadata: {}
|
87
87
|
post_install_message:
|
88
88
|
rdoc_options: []
|