nesquikcsv 0.1.6-java → 0.1.7-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 255038dee22845b62044b5fe0df5631cc6379ce5
-  data.tar.gz: cb66e0db4ef59b7f466c6127d45e0420938556e2
+  metadata.gz: fbc244bfa0588d12fcfa0254677cb456884544f2
+  data.tar.gz: d47f2aae0c71b27494df777ba41649c47666d1d6
 SHA512:
-  metadata.gz: 80879dc0e7b5ade30f51f6cbbb1980f190e11115f4c15ba65a78a294f79638efe6e20eed54259f34f8670559529b79c21af0ea0ca72b6ef734d6395811240283
-  data.tar.gz: fca3005766cf84bde3c23c01613a61074db0f9d5981e7e655602f9f896243e3da41ca26ee95a92dac7578a74b67b8c4c1402117e2f15c653fcf1fd5581dda422
+  metadata.gz: 9bb5895ae76a051b8740d6641190e00914181ada5701f51bcc80eb8a64ce11c6d597cfd1a43b20e14de1a6403dcf54d7a39b3c0d5277301114771f90d49bb302
+  data.tar.gz: a621bcbe880b777d2edaa23b15dd76187ef1bac1bff38a555722e6a1f7226da0c585d32f349c29be02dc88bb8e7540c4c799f7bbfc979544846d8f7732b3b392

data/README.md

@@ -2,7 +2,7 @@
 
 Fork of the Fastest-CSV gem to support any encoding, using UTF-8 by default.
 
-Uses native C code to parse CSV lines in MRI Ruby.
+Uses native C code to parse CSV lines in MRI Ruby, falls back to standard library CSV parser for JRuby
 
 Supports standard CSV according to RFC4180. Not the so-called "csv" from Excel.
 

data/Rakefile

@@ -4,7 +4,8 @@ require "bundler/gem_tasks"
 spec = Gem::Specification.load('nesquikcsv.gemspec')
 
 if RUBY_PLATFORM =~ /java/
-  # TODO
+  require 'rake/javaextensiontask'
+  Rake::JavaExtensionTask.new('csv_parser', spec)
 else
   require 'rake/extensiontask'
   Rake::ExtensionTask.new('csv_parser', spec)
@@ -27,3 +28,4 @@ Rake::TestTask.new do |t|
   #test.verbose = true
 end
 
+task :default => [:test]

data/ext/csv_parser/CsvParser.java

@@ -0,0 +1,89 @@
+//
+// Copyright (c) Maarten Oelering, BrightCode BV
+//
+
+package org.brightcode;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public class CsvParser {
+
+    private static int DEF_ARRAY_LEN = 32;
+
+    private static int UNQUOTED = 0;
+    private static int IN_QUOTED = 1;
+    private static int QUOTE_IN_QUOTED = 2;
+    
+    public static List parseLine(String line, String encoding) {
+        int length = line.length();
+        if (length == 0)
+            return null;
+        
+        int state = UNQUOTED;
+        StringBuilder value = new StringBuilder(length);   // field value, no longer than line
+        List<String> array = new ArrayList<String>(DEF_ARRAY_LEN);
+        
+        for (int i = 0; i < length; i++) {
+            char c = line.charAt(i);
+            switch (c) {
+                case ',':
+                    if (state == UNQUOTED) {
+                        if (value.length() == 0) {
+                            array.add(null);
+                        } 
+                        else {
+                            array.add(value.toString());
+                            value.setLength(0);
+                        }
+                    }
+                    else if (state == IN_QUOTED) {
+                        value.append(c);
+                    }
+                    else if (state == QUOTE_IN_QUOTED) {
+                        array.add(value.toString());
+                        value.setLength(0);
+                        state = UNQUOTED;
+                    }
+                    break;
+                case '"':
+                    if (state == UNQUOTED) {
+                        state = IN_QUOTED;
+                    }
+                    else if (state == IN_QUOTED) {
+                        state = QUOTE_IN_QUOTED;
+                    }
+                    else if (state == QUOTE_IN_QUOTED) {
+                        value.append(c);   // escaped quote
+                        state = IN_QUOTED;
+                    }
+                    break;
+                case '\r':
+                case '\n':
+                    if (state == IN_QUOTED) {
+                        value.append(c);
+                    }
+                    else {
+                        i = length;  // only parse first line if multiline
+                    }
+                    break;
+                default:
+                    value.append(c);
+                    break;
+            }
+        }
+        if (state == UNQUOTED) {
+            if (value.length() == 0) {
+                array.add(null);
+            } 
+            else {
+                array.add(value.toString());
+                value.setLength(0);
+            }
+        }
+        else if (state == QUOTE_IN_QUOTED) {
+            array.add(value.toString());
+        }
+        return array;
+    }
+}

data/lib/nesquikcsv/version.rb

@@ -1,3 +1,3 @@
 class NesquikCSV
-  VERSION = "0.1.6"
+  VERSION = "0.1.7"
 end

data/lib/nesquikcsv.rb

@@ -1,13 +1,13 @@
-if RUBY_PLATFORM =~ /java/
-  require 'csv'
-else
-  require 'csv_parser'
-end
+require 'csv_parser'
 require 'stringio'
 
 # Fast CSV parser using native code
 class NesquikCSV
   include Enumerable
+
+  if RUBY_PLATFORM =~ /java/
+    include_package "org.brightcode"
+  end
   
   # Pass each line of the specified +path+ as array to the provided +block+
   def self.foreach(path, &block)
@@ -57,10 +57,9 @@ class NesquikCSV
   
   def self.parse_line(line, encoding="UTF-8")
     if RUBY_PLATFORM =~ /java/
-      CSV.parse_line(line.force_encoding(encoding))
-    else
-      CsvParser.parse_line(line, encoding)
+      line = line.force_encoding(encoding)
     end
+    CsvParser.parse_line(line, encoding)
   end
 
   # Create new NesquikCSV wrapping the specified IO object
@@ -96,11 +95,7 @@ class NesquikCSV
   # Read next line from the wrapped IO and return as array or nil at EOF
   def shift(encoding='UTF-8')
     if line = get_line_with_quotes
-      if RUBY_PLATFORM =~ /java/
-        CSV.parse_line(line.force_encoding(encoding))
-      else
-        CsvParser.parse_line(line, encoding)
-      end
+      CsvParser.parse_line(line, encoding)
     else
       nil
     end

data/nesquikcsv.gemspec

@@ -17,6 +17,7 @@ Gem::Specification.new do |gem|
 
   if RUBY_PLATFORM =~ /java/
     gem.platform = "java"
+    gem.files << "lib/csv_parser.jar"
   else
     gem.extensions  = ['ext/csv_parser/extconf.rb']
   end

data/test/tc_csv_parsing.rb

@@ -3,144 +3,141 @@
 # Tests copied from faster_csv by James Edward Gray II
 #
 
-if RUBY_DESCRIPTION =~ /jruby/
-  # TODO
-  # These tests dont make sense for jruby since there's no Java extension
-else
+require 'test/unit'
+require 'nesquikcsv'
 
-  require 'test/unit'
-  require 'nesquikcsv'
+# 
+# Following tests are my interpretation of the 
+# {CSV RCF}[http://www.ietf.org/rfc/rfc4180.txt].  I only deviate from that 
+# document in one place (intentionally) and that is to make the default row
+# separator <tt>$/</tt>.
+# 
+class TestCSVParsing < Test::Unit::TestCase
 
-  # 
-  # Following tests are my interpretation of the 
-  # {CSV RCF}[http://www.ietf.org/rfc/rfc4180.txt].  I only deviate from that 
-  # document in one place (intentionally) and that is to make the default row
-  # separator <tt>$/</tt>.
-  # 
-  class TestCSVParsing < Test::Unit::TestCase
-
-    def test_mastering_regex_example
-      ex = %Q{Ten Thousand,10000, 2710 ,,"10,000","It's ""10 Grand"", baby",10K}
-      assert_equal( [ "Ten Thousand", "10000", " 2710 ", nil, "10,000",
-                      "It's \"10 Grand\", baby", "10K" ],
-                    CsvParser.parse_line(ex, "UTF-8") )
-    end
-
-    # Pulled from:  http://www.ruby-lang.org/cgi-bin/cvsweb.cgi/ruby/test/csv/test_csv.rb?rev=1.12.2.2;content-type=text%2Fplain
-    def test_std_lib_csv
-      [ ["\t", ["\t"]],
-        ["foo,\"\"\"\"\"\",baz", ["foo", "\"\"", "baz"]],
-        ["foo,\"\"\"bar\"\"\",baz", ["foo", "\"bar\"", "baz"]],
-        ["\"\"\"\n\",\"\"\"\n\"", ["\"\n", "\"\n"]],
-        ["foo,\"\r\n\",baz", ["foo", "\r\n", "baz"]],
-        ["\"\"", [""]],
-        ["foo,\"\"\"\",baz", ["foo", "\"", "baz"]],
-        ["foo,\"\r.\n\",baz", ["foo", "\r.\n", "baz"]],
-        ["foo,\"\r\",baz", ["foo", "\r", "baz"]],
-        ["foo,\"\",baz", ["foo", "", "baz"]],
-        ["\",\"", [","]],
-        ["foo", ["foo"]],
-        [",,", [nil, nil, nil]],
-        [",", [nil, nil]],
-        ["foo,\"\n\",baz", ["foo", "\n", "baz"]],
-        ["foo,,baz", ["foo", nil, "baz"]],
-        ["\"\"\"\r\",\"\"\"\r\"", ["\"\r", "\"\r"]],
-        ["\",\",\",\"", [",", ","]],
-        ["foo,bar,", ["foo", "bar", nil]],
-        [",foo,bar", [nil, "foo", "bar"]],
-        ["foo,bar", ["foo", "bar"]],
-        [";", [";"]],
-        ["\t,\t", ["\t", "\t"]],
-        ["foo,\"\r\n\r\",baz", ["foo", "\r\n\r", "baz"]],
-        ["foo,\"\r\n\n\",baz", ["foo", "\r\n\n", "baz"]],
-        ["foo,\"foo,bar\",baz", ["foo", "foo,bar", "baz"]],
-        [";,;", [";", ";"]] ].each do |csv_test|
-        assert_equal(csv_test.last, CsvParser.parse_line(csv_test.first, "UTF-8"))
-      end
-      
-      [ ["foo,\"\"\"\"\"\",baz", ["foo", "\"\"", "baz"]],
-        ["foo,\"\"\"bar\"\"\",baz", ["foo", "\"bar\"", "baz"]],
-        ["foo,\"\r\n\",baz", ["foo", "\r\n", "baz"]],
-        ["\"\"", [""]],
-        ["foo,\"\"\"\",baz", ["foo", "\"", "baz"]],
-        ["foo,\"\r.\n\",baz", ["foo", "\r.\n", "baz"]],
-        ["foo,\"\r\",baz", ["foo", "\r", "baz"]],
-        ["foo,\"\",baz", ["foo", "", "baz"]],
-        ["foo", ["foo"]],
-        [",,", [nil, nil, nil]],
-        [",", [nil, nil]],
-        ["foo,\"\n\",baz", ["foo", "\n", "baz"]],
-        ["foo,,baz", ["foo", nil, "baz"]],
-        ["foo,bar", ["foo", "bar"]],
-        ["foo,\"\r\n\n\",baz", ["foo", "\r\n\n", "baz"]],
-        ["foo,\"foo,bar\",baz", ["foo", "foo,bar", "baz"]] ].each do |csv_test|
-        assert_equal(csv_test.last, CsvParser.parse_line(csv_test.first, "UTF-8"))
-       end
-    end
+  if RUBY_PLATFORM =~ /java/
+    include_package "org.brightcode"
+  end
 
-    # From:  http://ruby-talk.org/cgi-bin/scat.rb/ruby/ruby-core/6496
-    def test_aras_edge_cases
-      [ [%Q{a,b},               ["a", "b"]],
-        [%Q{a,"""b"""},         ["a", "\"b\""]],
-        [%Q{a,"""b"},           ["a", "\"b"]],
-        [%Q{a,"b"""},           ["a", "b\""]],
-        [%Q{a,"\nb"""},         ["a", "\nb\""]],
-        [%Q{a,"""\nb"},         ["a", "\"\nb"]],
-        [%Q{a,"""\nb\n"""},     ["a", "\"\nb\n\""]],
-        [%Q{a,"""\nb\n""",\nc}, ["a", "\"\nb\n\"", nil]],
-        [%Q{a,,,},              ["a", nil, nil, nil]],
-        [%Q{,},                 [nil, nil]],
-        [%Q{"",""},             ["", ""]],
-        [%Q{""""},              ["\""]],
-        [%Q{"""",""},           ["\"",""]],
-        [%Q{,""},               [nil,""]],
-        [%Q{,"\r"},             [nil,"\r"]],
-        [%Q{"\r\n,"},           ["\r\n,"]],
-        [%Q{"\r\n,",},          ["\r\n,", nil]] ].each do |edge_case|
-          assert_equal(edge_case.last, CsvParser.parse_line(edge_case.first, "UTF-8"))
-        end
-    end
+  def test_mastering_regex_example
+    ex = %Q{Ten Thousand,10000, 2710 ,,"10,000","It's ""10 Grand"", baby",10K}
+    assert_equal( [ "Ten Thousand", "10000", " 2710 ", nil, "10,000",
+                    "It's \"10 Grand\", baby", "10K" ],
+                  CsvParser.parse_line(ex, "UTF-8") )
+  end
 
-    def test_james_edge_cases
-      # A read at eof? should return nil.
-      assert_equal(nil, CsvParser.parse_line("", "UTF-8"))
-      # 
-      # With CSV it's impossible to tell an empty line from a line containing a
-      # single +nil+ field.  The standard CSV library returns <tt>[nil]</tt>
-      # in these cases, but <tt>Array.new</tt> makes more sense to me.
-      # 
-      #assert_equal(Array.new, NesquikCSV.parse_line("\n1,2,3\n"))
-      assert_equal([nil], CsvParser.parse_line("\n1,2,3\n", "UTF-8"))
+  # Pulled from:  http://www.ruby-lang.org/cgi-bin/cvsweb.cgi/ruby/test/csv/test_csv.rb?rev=1.12.2.2;content-type=text%2Fplain
+  def test_std_lib_csv
+    [ ["\t", ["\t"]],
+      ["foo,\"\"\"\"\"\",baz", ["foo", "\"\"", "baz"]],
+      ["foo,\"\"\"bar\"\"\",baz", ["foo", "\"bar\"", "baz"]],
+      ["\"\"\"\n\",\"\"\"\n\"", ["\"\n", "\"\n"]],
+      ["foo,\"\r\n\",baz", ["foo", "\r\n", "baz"]],
+      ["\"\"", [""]],
+      ["foo,\"\"\"\",baz", ["foo", "\"", "baz"]],
+      ["foo,\"\r.\n\",baz", ["foo", "\r.\n", "baz"]],
+      ["foo,\"\r\",baz", ["foo", "\r", "baz"]],
+      ["foo,\"\",baz", ["foo", "", "baz"]],
+      ["\",\"", [","]],
+      ["foo", ["foo"]],
+      [",,", [nil, nil, nil]],
+      [",", [nil, nil]],
+      ["foo,\"\n\",baz", ["foo", "\n", "baz"]],
+      ["foo,,baz", ["foo", nil, "baz"]],
+      ["\"\"\"\r\",\"\"\"\r\"", ["\"\r", "\"\r"]],
+      ["\",\",\",\"", [",", ","]],
+      ["foo,bar,", ["foo", "bar", nil]],
+      [",foo,bar", [nil, "foo", "bar"]],
+      ["foo,bar", ["foo", "bar"]],
+      [";", [";"]],
+      ["\t,\t", ["\t", "\t"]],
+      ["foo,\"\r\n\r\",baz", ["foo", "\r\n\r", "baz"]],
+      ["foo,\"\r\n\n\",baz", ["foo", "\r\n\n", "baz"]],
+      ["foo,\"foo,bar\",baz", ["foo", "foo,bar", "baz"]],
+      [";,;", [";", ";"]] ].each do |csv_test|
+      assert_equal(csv_test.last, CsvParser.parse_line(csv_test.first, "UTF-8"))
     end
+    
+    [ ["foo,\"\"\"\"\"\",baz", ["foo", "\"\"", "baz"]],
+      ["foo,\"\"\"bar\"\"\",baz", ["foo", "\"bar\"", "baz"]],
+      ["foo,\"\r\n\",baz", ["foo", "\r\n", "baz"]],
+      ["\"\"", [""]],
+      ["foo,\"\"\"\",baz", ["foo", "\"", "baz"]],
+      ["foo,\"\r.\n\",baz", ["foo", "\r.\n", "baz"]],
+      ["foo,\"\r\",baz", ["foo", "\r", "baz"]],
+      ["foo,\"\",baz", ["foo", "", "baz"]],
+      ["foo", ["foo"]],
+      [",,", [nil, nil, nil]],
+      [",", [nil, nil]],
+      ["foo,\"\n\",baz", ["foo", "\n", "baz"]],
+      ["foo,,baz", ["foo", nil, "baz"]],
+      ["foo,bar", ["foo", "bar"]],
+      ["foo,\"\r\n\n\",baz", ["foo", "\r\n\n", "baz"]],
+      ["foo,\"foo,bar\",baz", ["foo", "foo,bar", "baz"]] ].each do |csv_test|
+      assert_equal(csv_test.last, CsvParser.parse_line(csv_test.first, "UTF-8"))
+     end
+  end
 
-    def test_rob_edge_cases
-      [ [%Q{"a\nb"},                         ["a\nb"]],
-        [%Q{"\n\n\n"},                       ["\n\n\n"]],
-        [%Q{a,"b\n\nc"},                     ['a', "b\n\nc"]],
-        [%Q{,"\r\n"},                        [nil,"\r\n"]],
-        [%Q{,"\r\n."},                       [nil,"\r\n."]],
-        [%Q{"a\na","one newline"},           ["a\na", 'one newline']],
-        [%Q{"a\n\na","two newlines"},        ["a\n\na", 'two newlines']],
-        [%Q{"a\r\na","one CRLF"},            ["a\r\na", 'one CRLF']],
-        [%Q{"a\r\n\r\na","two CRLFs"},       ["a\r\n\r\na", 'two CRLFs']],
-        [%Q{with blank,"start\n\nfinish"\n}, ['with blank', "start\n\nfinish"]],
-      ].each do |edge_case|
+  # From:  http://ruby-talk.org/cgi-bin/scat.rb/ruby/ruby-core/6496
+  def test_aras_edge_cases
+    [ [%Q{a,b},               ["a", "b"]],
+      [%Q{a,"""b"""},         ["a", "\"b\""]],
+      [%Q{a,"""b"},           ["a", "\"b"]],
+      [%Q{a,"b"""},           ["a", "b\""]],
+      [%Q{a,"\nb"""},         ["a", "\nb\""]],
+      [%Q{a,"""\nb"},         ["a", "\"\nb"]],
+      [%Q{a,"""\nb\n"""},     ["a", "\"\nb\n\""]],
+      [%Q{a,"""\nb\n""",\nc}, ["a", "\"\nb\n\"", nil]],
+      [%Q{a,,,},              ["a", nil, nil, nil]],
+      [%Q{,},                 [nil, nil]],
+      [%Q{"",""},             ["", ""]],
+      [%Q{""""},              ["\""]],
+      [%Q{"""",""},           ["\"",""]],
+      [%Q{,""},               [nil,""]],
+      [%Q{,"\r"},             [nil,"\r"]],
+      [%Q{"\r\n,"},           ["\r\n,"]],
+      [%Q{"\r\n,",},          ["\r\n,", nil]] ].each do |edge_case|
         assert_equal(edge_case.last, CsvParser.parse_line(edge_case.first, "UTF-8"))
       end
-    end
+  end
 
-    def test_encoding
-      assert_equal(["ñ","ó","¸"], CsvParser.parse_line("ñ,ó,¸", "UTF-8"))
-    end
+  def test_james_edge_cases
+    # A read at eof? should return nil.
+    assert_equal(nil, CsvParser.parse_line("", "UTF-8"))
+    # 
+    # With CSV it's impossible to tell an empty line from a line containing a
+    # single +nil+ field.  The standard CSV library returns <tt>[nil]</tt>
+    # in these cases, but <tt>Array.new</tt> makes more sense to me.
+    # 
+    #assert_equal(Array.new, NesquikCSV.parse_line("\n1,2,3\n"))
+    assert_equal([nil], CsvParser.parse_line("\n1,2,3\n", "UTF-8"))
+  end
 
-    def test_quoted_line_break
-      assert_equal(["foo","bar,baz\nbeam","bee"], CsvParser.parse_line("foo,\"bar,baz\nbeam\",bee", "UTF-8"))
+  def test_rob_edge_cases
+    [ [%Q{"a\nb"},                         ["a\nb"]],
+      [%Q{"\n\n\n"},                       ["\n\n\n"]],
+      [%Q{a,"b\n\nc"},                     ['a', "b\n\nc"]],
+      [%Q{,"\r\n"},                        [nil,"\r\n"]],
+      [%Q{,"\r\n."},                       [nil,"\r\n."]],
+      [%Q{"a\na","one newline"},           ["a\na", 'one newline']],
+      [%Q{"a\n\na","two newlines"},        ["a\n\na", 'two newlines']],
+      [%Q{"a\r\na","one CRLF"},            ["a\r\na", 'one CRLF']],
+      [%Q{"a\r\n\r\na","two CRLFs"},       ["a\r\n\r\na", 'two CRLFs']],
+      [%Q{with blank,"start\n\nfinish"\n}, ['with blank', "start\n\nfinish"]],
+    ].each do |edge_case|
+      assert_equal(edge_case.last, CsvParser.parse_line(edge_case.first, "UTF-8"))
     end
+  end
 
-    def test_quoted_line_break_at_end
-      assert_equal(["foo","bar,baz\n","bee"], CsvParser.parse_line("foo,\"bar,baz\n\",bee", "UTF-8"))
-    end
+  def test_encoding
+    assert_equal(["ñ","ó","¸"], CsvParser.parse_line("ñ,ó,¸", "UTF-8"))
+  end
+
+  def test_quoted_line_break
+    assert_equal(["foo","bar,baz\nbeam","bee"], CsvParser.parse_line("foo,\"bar,baz\nbeam\",bee", "UTF-8"))
+  end
 
+  def test_quoted_line_break_at_end
+    assert_equal(["foo","bar,baz\n","bee"], CsvParser.parse_line("foo,\"bar,baz\n\",bee", "UTF-8"))
   end
 
 end

metadata

@@ -1,43 +1,43 @@
 --- !ruby/object:Gem::Specification
 name: nesquikcsv
 version: !ruby/object:Gem::Version
-  version: 0.1.6
+  version: 0.1.7
 platform: java
 authors:
 - Juan Martty
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2017-01-02 00:00:00.000000000 Z
+date: 2017-01-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: test-unit
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - '>='
-      - !ruby/object:Gem::Version
-        version: '0'
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
+  name: test-unit
   prerelease: false
   type: :development
-- !ruby/object:Gem::Dependency
-  name: rake-compiler
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
+  name: rake-compiler
   prerelease: false
   type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 description: Fastest-CSV fork with encoding support
 email:
 - null.terminated.string@gmail.com
@@ -50,8 +50,10 @@ files:
 - LICENSE
 - README.md
 - Rakefile
+- ext/csv_parser/CsvParser.java
 - ext/csv_parser/extconf.rb
 - ext/csv_parser/parser.c
+- lib/csv_parser.jar
 - lib/nesquikcsv.rb
 - lib/nesquikcsv/version.rb
 - nesquikcsv.gemspec