csv_lazy 0.0.4 → 0.0.5

data/Gemfile

@@ -3,6 +3,8 @@ source "http://rubygems.org"
 # Example:
 #   gem "activesupport", ">= 2.3.5"
 
+gem "string_utils"
+
 # Add dependencies to develop your gem here.
 # Include everything needed to run rake, tests, features, etc.
 group :development do

data/Gemfile.lock

@@ -20,6 +20,7 @@ GEM
     rspec-expectations (2.8.0)
       diff-lcs (~> 1.1.2)
     rspec-mocks (2.8.0)
+    string_utils ()
 
 PLATFORMS
   ruby
@@ -29,3 +30,4 @@ DEPENDENCIES
   jeweler (~> 1.8.4)
   rdoc (~> 3.12)
   rspec (~> 2.8.0)
+  string_utils

data/VERSION

@@ -1 +1 @@
-0.0.4
+0.0.5

data/csv_lazy.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = "csv_lazy"
-  s.version = "0.0.4"
+  s.version = "0.0.5"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Kasper Johansen"]
-  s.date = "2013-03-22"
+  s.date = "2013-03-25"
   s.description = "A small CSV lib that skips whitespace-format-bugs and more."
   s.email = "k@spernj.org"
   s.extra_rdoc_files = [
@@ -29,7 +29,8 @@ Gem::Specification.new do |s|
     "lib/csv_lazy.rb",
     "spec/csv_lazy_spec.rb",
     "spec/spec_helper.rb",
-    "spec/test1.csv.gz"
+    "spec/test1.csv.gz",
+    "spec/test2.csv"
   ]
   s.homepage = "http://github.com/kaspernj/csv_lazy"
   s.licenses = ["MIT"]
@@ -41,17 +42,20 @@ Gem::Specification.new do |s|
     s.specification_version = 3
 
     if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
+      s.add_runtime_dependency(%q<string_utils>, [">= 0"])
       s.add_development_dependency(%q<rspec>, ["~> 2.8.0"])
       s.add_development_dependency(%q<rdoc>, ["~> 3.12"])
       s.add_development_dependency(%q<bundler>, [">= 1.0.0"])
       s.add_development_dependency(%q<jeweler>, ["~> 1.8.4"])
     else
+      s.add_dependency(%q<string_utils>, [">= 0"])
       s.add_dependency(%q<rspec>, ["~> 2.8.0"])
       s.add_dependency(%q<rdoc>, ["~> 3.12"])
       s.add_dependency(%q<bundler>, [">= 1.0.0"])
       s.add_dependency(%q<jeweler>, ["~> 1.8.4"])
     end
   else
+    s.add_dependency(%q<string_utils>, [">= 0"])
     s.add_dependency(%q<rspec>, ["~> 2.8.0"])
     s.add_dependency(%q<rdoc>, ["~> 3.12"])
     s.add_dependency(%q<bundler>, [">= 1.0.0"])

data/lib/csv_lazy.rb

@@ -1,3 +1,6 @@
+#encoding: utf-8
+require "string_utils"
+
 #A simple library for parsing CSV-files through IO's. Solves corrupt file formats automatically like when files contains several spaces after a column and more.
 class Csv_lazy
   include Enumerable
@@ -13,7 +16,8 @@ class Csv_lazy
       :quote_char => '"',
       :row_sep => "\n",
       :col_sep => ";",
-      :headers => false
+      :headers => false,
+      :buffer_length => 4096
     }.merge(args)
     
     @io = @args[:io]
@@ -22,9 +26,14 @@ class Csv_lazy
     @debug = @args[:debug]
     @encode = @args[:encode]
     @mutex = Mutex.new
+    @buffer_length = @args[:buffer_length]
+    @escape_char = "\\"
+    @escaped_quote = "#{@escape_char}#{@args[:quote_char]}"
+    @escaped_quote_double = "#{@escape_char}#{@escape_char}#{@args[:quote_char]}"
+    
     #@debug = true
     
-    accepted = [:encode, :quote_char, :row_sep, :col_sep, :io, :debug, :headers]
+    accepted = [:encode, :quote_char, :row_sep, :col_sep, :io, :debug, :headers, :buffer_length]
     @args.each do |key, val|
       if accepted.index(key) == nil
         raise "Unknown argument: '#{key}'."
@@ -60,24 +69,20 @@ class Csv_lazy
   
   #Yields each row as an array.
   def each
-    @mutex.synchronize do
-      while row = read_row
-        yield(row)
+    if block_given?
+      @mutex.synchronize do
+        while row = read_row
+          yield(row)
+        end
       end
-    end
-  end
-  
-  private
-  
-  #Reads more content into the buffer.
-  def read_buffer
-    read = @io.gets
-    
-    if !read
-      @eof = true
     else
-      read = read.encode(@encode) if @encode
-      @buffer << read
+      Enumerable.new do |yielder|
+        @mutex.synchronize do
+          while row = read_row
+            yielder << row
+          end
+        end
+      end
     end
   end
   
@@ -109,6 +114,20 @@ class Csv_lazy
     end
   end
   
+  private
+  
+  #Reads more content into the buffer.
+  def read_buffer
+    read = @io.gets
+    
+    if !read
+      @eof = true
+    else
+      read = read.encode(@encode) if @encode
+      @buffer << read
+    end
+  end
+  
   #Runs a regex against the buffer. If matched it also removes it from the buffer.
   def read_remove_regex(regex)
     if match = @buffer.match(regex)
@@ -130,23 +149,50 @@ class Csv_lazy
     return false
   end
   
+  def unescape(str)
+    return StringUtils.strtr(str, {
+      "\\\\" => "\\",
+      "\\t" => "\t",
+      "\\n" => "\n",
+      "\\\"" => "\""
+    })
+  end
+  
   #Adds the next column to the row. Returns true if more columns should be read or false if this was the end of the row.
   def read_next_col
-    read_buffer if @buffer.length < 4096
+    read_buffer if @buffer.length < @buffer_length
     return false if @buffer.empty? and @eof
     
     if @buffer.empty? or read_remove_regex(@regex_row_end)
       return false
     elsif match = read_remove_regex(@regex_begin_quote_char)
       read = ""
+      col_content = ""
       
       loop do
         match_read = read_remove_regex(@regex_read_until_quote_char)
         if !match_read
-          read_buffer
+          if @eof
+            add_col(@buffer) unless @buffer.empty?
+            @buffer = ""
+            break
+          else
+            read_buffer
+          end
         else
-          add_col(match_read[1])
-          break
+          all = match_read[0]
+          escaped_quote_char = all[-@escaped_quote.length, @escaped_quote.length]
+          double_escaped_quote_char = all[-@escaped_quote_double.length, @escaped_quote_double.length]
+          all_without_quote = match_read[1]
+          
+          if escaped_quote_char == @escaped_quote and double_escaped_quote_char != @escaped_quote_double
+            #continue reading - the quote char is escaped.
+            col_content << all
+          else
+            col_content << match_read[1]
+            add_col(unescape(col_content))
+            break
+          end
         end
       end
       
@@ -182,9 +228,10 @@ class Csv_lazy
       read_buffer
       raise Errno::EAGAIN
     else
-      raise "Dont know what to do with buffer: #{@buffer}"
+      raise "Dont know what to do with buffer: '#{@buffer}'."
     end
   rescue Errno::EAGAIN
+    puts "csv_lazy: Retry! Probably we ran out of buffer..." if @debug
     retry
   end
   

data/spec/csv_lazy_spec.rb

@@ -86,8 +86,8 @@ describe "CsvLazy" do
   
   it "should be able to use headers and return hashes instead" do
     cont = "\"name\",age\r\n"
-    cont += "\"Kasper Johansen\",27\r\n"
-    cont += "\"Christina Stoeckel\",\"25\"\r\n"
+    cont << "\"Kasper Johansen\",27\r\n"
+    cont << "\"Christina Stoeckel\",\"25\"\r\n"
     
     line = 0
     Csv_lazy.new(:col_sep => ",", :io => StringIO.new(cont), :headers => true, :row_sep => "\r\n") do |csv|
@@ -109,4 +109,39 @@ describe "CsvLazy" do
     
     line.should eql(2)
   end
+  
+  it "should be able to encode incoming strings from weird files without crashing" do
+    File.open("#{File.dirname(__FILE__)}/test2.csv", "rb", :encoding => "UTF-16LE") do |fp|
+      #Remove invalid UTF content.
+      fp.read(2)
+      
+      Csv_lazy.new(:col_sep => ",", :io => fp, :headers => true, :row_sep => "\r\n", :quote_char => '"', :encode => "US-ASCII", :debug => false) do |csv|
+        csv.keys[0].should eql(:legacy_user_id)
+        csv.keys[1].should eql(:savings_percentage)
+        csv.keys[2].should eql(:active)
+        csv.keys.length.should eql(3)
+      end
+    end
+  end
+  
+  it "should do proper escaping" do
+    cont = "\"Test1\";\"Test2 \\\"Wee\\\"\"\r\n"
+    cont << "\"Test3\";\"Test4 \\\"Wee\\\"\";\"Test5 \\\"Wee\\\"\"\r\n"
+    
+    csv = Csv_lazy.new(:col_sep => ";", :io => StringIO.new(cont), :row_sep => "\r\n")
+    
+    row = csv.read_row
+    row[0].should eql("Test1")
+    row[1].should eql("Test2 \"Wee\"")
+    row.length.should eql(2)
+    
+    row = csv.read_row
+    row[0].should eql("Test3")
+    row[1].should eql("Test4 \"Wee\"")
+    row[2].should eql("Test5 \"Wee\"")
+    row.length.should eql(3)
+    
+    row = csv.read_row
+    row.should eql(false)
+  end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: csv_lazy
 version: !ruby/object:Gem::Version
-  version: 0.0.4
+  version: 0.0.5
   prerelease: 
 platform: ruby
 authors:
@@ -9,8 +9,24 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-03-22 00:00:00.000000000 Z
+date: 2013-03-25 00:00:00.000000000 Z
 dependencies:
+- !ruby/object:Gem::Dependency
+  name: string_utils
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
@@ -96,6 +112,7 @@ files:
 - spec/csv_lazy_spec.rb
 - spec/spec_helper.rb
 - spec/test1.csv.gz
+- spec/test2.csv
 homepage: http://github.com/kaspernj/csv_lazy
 licenses:
 - MIT
@@ -111,7 +128,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: 222799097840594694
+      hash: -1390404400491179462
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements: