filter_io 0.2.2 → 0.2.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 9208b4c2d0df82cc91ed1cad2283a3d36218987c
-  data.tar.gz: a577f773a20e4e2be7bd219b01629d020a7522dc
+  metadata.gz: 153015f29c925bdc3e3f232bef84b24ce66e5cd5
+  data.tar.gz: 7f67801d371b9e05b6b543ff3cc9268e8e96868d
 SHA512:
-  metadata.gz: 8edcd67f5b6220499dd21c6453d695eee991e8770f3ea48787fc1c72433275d8efe49c68fd27c48d0208edd76ad5bdf33e4fa612cc53e8a9f64e411a2348729d
-  data.tar.gz: ea4b663fbd4aca1f2d85259904c008ca30c23ee409c76876f039aa9120115c07ef0d8b5fffe58107b39f97de150bd9310539307bf3ea232b782fa57ac6bcb53f
+  metadata.gz: 16952dba9a57805e96fb8f23f78bab8ea9b2b2f4638a5eda2016a8119793515066320b9d254b2008e2154f4f62bb920d572b7d8862460354d8a9443176765256
+  data.tar.gz: 8d3f368facfd069549d6d6881ec6643616a0fd6a47bac65589eef74a3f53d2d6766e51f98436c3800d0edfaa3acd36f780f53d9b6716c58a5c65642a837e5b50

data/README.markdown

@@ -30,8 +30,10 @@ end
 A common usage of `filter_io` is to normalise line endings before parsing CSV data:
 
 ``` ruby
+require 'csv'
+
 # open source stream
-File.open(filename) do |io|
+File.open(filename, external_encoding: 'UTF-8') do |io|
   # apply filter to stream
   io = FilterIO.new(io) do |data, state|
     # grab another chunk if the last character is a delimiter
@@ -39,10 +41,10 @@ File.open(filename) do |io|
     # normalise line endings to LF
     data.gsub /\r\n|\r|\n/, "\n"
   end
-  
+
   # process resulting stream normally
-  FasterCSV.parse(io) do |row|
-    pp row
+  CSV.parse(io, row_sep: "\n") do |row|
+    p row
   end
 end
 ```

data/Rakefile

@@ -4,3 +4,8 @@ require 'rspec/core/rake_task'
 
 RSpec::Core::RakeTask.new(:spec)
 task :default => :spec
+
+desc 'Open a Pry console with environment'
+task :console do
+  exec "pry -Ilib -rfilter_io"
+end

data/filter_io.gemspec

@@ -23,4 +23,5 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency 'rake'
   spec.add_development_dependency 'simplecov'
   spec.add_development_dependency 'rspec', '~> 2.13'
+  spec.add_development_dependency 'pry'
 end

data/lib/filter_io/version.rb

@@ -1,3 +1,3 @@
 class FilterIO
-  VERSION = '0.2.2'
+  VERSION = '0.2.3'
 end

data/lib/filter_io.rb

@@ -125,6 +125,7 @@ class FilterIO
       data
     when source_eof?
       # end of file, nothing in the buffer to return
+      buffer.replace empty_string if buffer
       length.nil? ? empty_string : nil
     else
       raise IOError, 'Read error'
@@ -169,7 +170,22 @@ class FilterIO
     @buffer = char + @buffer
   end
 
-  def gets(sep_string = $/)
+  def gets(*args)
+    sep_string, limit = case args.size
+    when 0
+      [$/, nil]
+    when 1
+      if args.first.is_a?(Integer)
+        [$/, args.first]
+      else
+        [args.first, nil]
+      end
+    when 2
+      args
+    else
+      raise ArgumentError
+    end
+
     return nil if eof?
     return read if sep_string.nil?
 
@@ -189,21 +205,29 @@ class FilterIO
     end
 
     # fill the buffer until it contains the separator sequence
-    until source_eof? or @buffer.index(sep_string)
+    until source_eof? || find_bytes(sep_string) || (limit && @buffer.bytesize >= limit)
       buffer_data @options[:block_size]
     end
 
     # calculate how much of the buffer to return
-    length = if idx = @buffer.index(sep_string)
+    length = if idx = find_bytes(sep_string)
       # we found the separator, include it in our output
-      length = idx + sep_string.size
+      length = idx + sep_string.bytesize
     else
       # no separator found (must be EOF). return everything we've got
-      length = @buffer.size
+      length = @buffer.bytesize
+    end
+    if limit && length > limit
+      length = limit
     end
 
-    # increment the position and return the buffer fragment
-    data = @buffer.slice!(0, length)
+    # extract the requested number of byte from the buffer
+    data = pop_bytes(length).force_encoding(@buffer.encoding)
+    # continue retreiving more bytes until we have complete characters
+    while limit && !data.valid_encoding? && (@buffer.bytesize > 0 || !source_eof?)
+      data += pop_bytes(1).force_encoding(@buffer.encoding)
+    end
+    # increment the position
     @pos += data.bytesize
 
     data
@@ -249,18 +273,30 @@ class FilterIO
     str
   end
 
-  def pop_bytes(count, output_buffer = nil)
+  def with_byte_buffer
     begin
       org_encoding = @buffer.encoding
       @buffer.force_encoding 'ASCII-8BIT'
+      yield
+    ensure
+      @buffer.force_encoding org_encoding
+    end
+  end
+
+  def pop_bytes(count, output_buffer = nil)
+    with_byte_buffer do
       data = @buffer.slice!(0, count)
       if output_buffer
         output_buffer.replace data
       else
         data
       end
-    ensure
-      @buffer.force_encoding org_encoding
+    end
+  end
+
+  def find_bytes(str)
+    with_byte_buffer do
+      @buffer.index(str)
     end
   end
 

data/spec/filter_io_spec.rb

@@ -4,6 +4,7 @@ require 'spec_helper'
 require 'stringio'
 require 'tempfile'
 require 'zlib'
+require 'csv'
 
 describe FilterIO do
   def matches_reference_io_behaviour(input)
@@ -36,6 +37,9 @@ describe FilterIO do
     # compare the filtered output against the reference
     results[0].zip(results[1]).each do |expected, actual|
       expect(actual).to eq expected
+      if actual.respond_to? :encoding
+        expect(actual.encoding).to eq expected.encoding
+      end
     end
   end
 
@@ -314,7 +318,13 @@ describe FilterIO do
     buffer = 'abcdef'
     result = io.read(3, buffer)
     expect(result.object_id).to eq buffer.object_id
-    expect(result).to eq 'foo'
+    expect(buffer).to eq 'foo'
+    result = io.read(4, buffer)
+    expect(result.object_id).to eq buffer.object_id
+    expect(buffer).to eq ' bar'
+    result = io.read(3, buffer)
+    expect(result).to eq nil
+    expect(buffer).to eq ''
   end
 
   it 'allows filtering of input with a block' do
@@ -540,6 +550,44 @@ describe FilterIO do
     end
   end
 
+  it 'supports `get` with a limit' do
+    [
+      "",
+      "x",
+      "foo\nbar\rbaz\n",
+      "abc\rdef\rghi\r",
+      "über",
+    ].each do |input|
+      [1, 2, 3, 4, 10].each do |limit|
+        matches_reference_io_behaviour(input) { |io| io.gets(limit) }
+      end
+    end
+    # TODO: test zero limit
+  end
+
+  it 'supports `gets` with a separator and a limit' do
+    [
+      "",
+      "x",
+      "foo\nbar\rbaz\n",
+      "abc\rdef\rghi\r",
+      "über",
+    ].each do |input|
+      ["\r", "x"].each do |sep_string|
+        [1, 2, 3, 4, 10].each do |limit|
+          matches_reference_io_behaviour(input) { |io| io.gets(sep_string, limit) }
+        end
+      end
+    end
+    # TODO: test zero limit
+  end
+
+  it 'errors when `get` is passed more than two args' do
+    expect {
+      FilterIO.new(StringIO.new).gets(1,2,3)
+    }.to raise_error ArgumentError
+  end
+
   it 'supports `gets` with a two character seperator' do
     ["o", "oo"].each do |sep_string|
       matches_reference_io_behaviour("foobarhelloworld") { |io| io.gets(sep_string) }
@@ -830,4 +878,19 @@ describe FilterIO do
     output.rewind
     expect(output.read).to eq 'TEST'
   end
+
+  it 'supports CSV' do
+    input = StringIO.new "foo,bar\nbaz"
+
+    filtered_input = FilterIO.new input do |data|
+      data.upcase
+    end
+
+    rows = []
+    CSV.parse(filtered_input) do |row|
+      rows << row
+    end
+
+    expect(rows).to eq [%w[FOO BAR], %w[BAZ]]
+  end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: filter_io
 version: !ruby/object:Gem::Version
-  version: 0.2.2
+  version: 0.2.3
 platform: ruby
 authors:
 - Jason Weathered
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-04-18 00:00:00.000000000 Z
+date: 2014-06-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -80,6 +80,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '2.13'
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: 
 email:
 - jason@jasoncodes.com