rw_fastercsv 1.5.6

data/test/export.csv

@@ -0,0 +1,5 @@
+Email Address,First Name,Last Name
+"www@example.com",,
+"bank@example.com","John J. Daw, Sr. and Mary Ann Daw","Daw"
+"z@example.com",,
+

data/test/tc_csv_parsing.rb

@@ -0,0 +1,194 @@
+#!/usr/local/bin/ruby -w
+
+# tc_csv_parsing.rb
+#
+#  Created by James Edward Gray II on 2005-10-31.
+#  Copyright 2005 Gray Productions. All rights reserved.
+
+require "test/unit"
+
+require "faster_csv"
+
+# 
+# Following tests are my interpretation of the 
+# {CSV RCF}[http://www.ietf.org/rfc/rfc4180.txt].  I only deviate from that 
+# document in one place (intentionally) and that is to make the default row
+# separator <tt>$/</tt>.
+# 
+class TestCSVParsing < Test::Unit::TestCase
+  def test_mastering_regex_example
+    ex = %Q{Ten Thousand,10000, 2710 ,,"10,000","It's ""10 Grand"", baby",10K}
+    assert_equal( [ "Ten Thousand", "10000", " 2710 ", nil, "10,000",
+                    "It's \"10 Grand\", baby", "10K" ],
+                  FasterCSV.parse_line(ex) )
+  end
+  
+  # Pulled from:  http://www.ruby-lang.org/cgi-bin/cvsweb.cgi/ruby/test/csv/test_csv.rb?rev=1.12.2.2;content-type=text%2Fplain
+  def test_std_lib_csv
+    [ ["\t", ["\t"]],
+      ["foo,\"\"\"\"\"\",baz", ["foo", "\"\"", "baz"]],
+      ["foo,\"\"\"bar\"\"\",baz", ["foo", "\"bar\"", "baz"]],
+      ["\"\"\"\n\",\"\"\"\n\"", ["\"\n", "\"\n"]],
+      ["foo,\"\r\n\",baz", ["foo", "\r\n", "baz"]],
+      ["\"\"", [""]],
+      ["foo,\"\"\"\",baz", ["foo", "\"", "baz"]],
+      ["foo,\"\r.\n\",baz", ["foo", "\r.\n", "baz"]],
+      ["foo,\"\r\",baz", ["foo", "\r", "baz"]],
+      ["foo,\"\",baz", ["foo", "", "baz"]],
+      ["\",\"", [","]],
+      ["foo", ["foo"]],
+      [",,", [nil, nil, nil]],
+      [",", [nil, nil]],
+      ["foo,\"\n\",baz", ["foo", "\n", "baz"]],
+      ["foo,,baz", ["foo", nil, "baz"]],
+      ["\"\"\"\r\",\"\"\"\r\"", ["\"\r", "\"\r"]],
+      ["\",\",\",\"", [",", ","]],
+      ["foo,bar,", ["foo", "bar", nil]],
+      [",foo,bar", [nil, "foo", "bar"]],
+      ["foo,bar", ["foo", "bar"]],
+      [";", [";"]],
+      ["\t,\t", ["\t", "\t"]],
+      ["foo,\"\r\n\r\",baz", ["foo", "\r\n\r", "baz"]],
+      ["foo,\"\r\n\n\",baz", ["foo", "\r\n\n", "baz"]],
+      ["foo,\"foo,bar\",baz", ["foo", "foo,bar", "baz"]],
+      [";,;", [";", ";"]] ].each do |csv_test|
+      assert_equal(csv_test.last, FasterCSV.parse_line(csv_test.first))
+    end
+
+    [ ["foo,\"\"\"\"\"\",baz", ["foo", "\"\"", "baz"]],
+      ["foo,\"\"\"bar\"\"\",baz", ["foo", "\"bar\"", "baz"]],
+      ["foo,\"\r\n\",baz", ["foo", "\r\n", "baz"]],
+      ["\"\"", [""]],
+      ["foo,\"\"\"\",baz", ["foo", "\"", "baz"]],
+      ["foo,\"\r.\n\",baz", ["foo", "\r.\n", "baz"]],
+      ["foo,\"\r\",baz", ["foo", "\r", "baz"]],
+      ["foo,\"\",baz", ["foo", "", "baz"]],
+      ["foo", ["foo"]],
+      [",,", [nil, nil, nil]],
+      [",", [nil, nil]],
+      ["foo,\"\n\",baz", ["foo", "\n", "baz"]],
+      ["foo,,baz", ["foo", nil, "baz"]],
+      ["foo,bar", ["foo", "bar"]],
+      ["foo,\"\r\n\n\",baz", ["foo", "\r\n\n", "baz"]],
+      ["foo,\"foo,bar\",baz", ["foo", "foo,bar", "baz"]] ].each do |csv_test|
+      assert_equal(csv_test.last, FasterCSV.parse_line(csv_test.first))
+    end
+  end
+  
+  # From:  http://ruby-talk.org/cgi-bin/scat.rb/ruby/ruby-core/6496
+  def test_aras_edge_cases
+    [ [%Q{a,b},               ["a", "b"]],
+      [%Q{a,"""b"""},         ["a", "\"b\""]],
+      [%Q{a,"""b"},           ["a", "\"b"]],
+      [%Q{a,"b"""},           ["a", "b\""]],
+      [%Q{a,"\nb"""},         ["a", "\nb\""]],
+      [%Q{a,"""\nb"},         ["a", "\"\nb"]],
+      [%Q{a,"""\nb\n"""},     ["a", "\"\nb\n\""]],
+      [%Q{a,"""\nb\n""",\nc}, ["a", "\"\nb\n\"", nil]],
+      [%Q{a,,,},              ["a", nil, nil, nil]],
+      [%Q{,},                 [nil, nil]],
+      [%Q{"",""},             ["", ""]],
+      [%Q{""""},              ["\""]],
+      [%Q{"""",""},           ["\"",""]],
+      [%Q{,""},               [nil,""]],
+      [%Q{,"\r"},             [nil,"\r"]],
+      [%Q{"\r\n,"},           ["\r\n,"]],
+      [%Q{"\r\n,",},          ["\r\n,", nil]] ].each do |edge_case|
+        assert_equal(edge_case.last, FasterCSV.parse_line(edge_case.first))
+      end
+  end
+  
+  def test_james_edge_cases
+    # A read at eof? should return nil.
+    assert_equal(nil, FasterCSV.parse_line(""))
+    # 
+    # With CSV it's impossible to tell an empty line from a line containing a
+    # single +nil+ field.  The standard CSV library returns <tt>[nil]</tt>
+    # in these cases, but <tt>Array.new</tt> makes more sense to me.
+    # 
+    assert_equal(Array.new, FasterCSV.parse_line("\n1,2,3\n"))
+  end
+
+  def test_rob_edge_cases
+    [ [%Q{"a\nb"},                         ["a\nb"]],
+      [%Q{"\n\n\n"},                       ["\n\n\n"]],
+      [%Q{a,"b\n\nc"},                     ['a', "b\n\nc"]],
+      [%Q{,"\r\n"},                        [nil,"\r\n"]],
+      [%Q{,"\r\n."},                       [nil,"\r\n."]],
+      [%Q{"a\na","one newline"},           ["a\na", 'one newline']],
+      [%Q{"a\n\na","two newlines"},        ["a\n\na", 'two newlines']],
+      [%Q{"a\r\na","one CRLF"},            ["a\r\na", 'one CRLF']],
+      [%Q{"a\r\n\r\na","two CRLFs"},       ["a\r\n\r\na", 'two CRLFs']],
+      [%Q{with blank,"start\n\nfinish"\n}, ['with blank', "start\n\nfinish"]],
+    ].each do |edge_case|
+      assert_equal(edge_case.last, FasterCSV.parse_line(edge_case.first))
+    end
+  end
+
+  def test_non_regex_edge_cases
+    # An early version of the non-regex parser fails this test
+    [["foo,\"foo,bar,baz,foo\",\"foo\"", ["foo", "foo,bar,baz,foo", "foo"]]].each do |edge_case|
+      assert_equal(edge_case.last, FasterCSV.parse_line(edge_case.first))
+    end
+
+    assert_raise(FasterCSV::MalformedCSVError) do
+      FasterCSV.parse_line("1,\"23\"4\"5\", 6")
+    end
+  end
+  
+  def test_malformed_csv
+    assert_raise(FasterCSV::MalformedCSVError) do
+      FasterCSV.parse_line("1,2\r,3", :row_sep => "\n")
+    end
+    
+    bad_data = <<-END_DATA.gsub(/^ +/, "")
+    line,1,abc
+    line,2,"def\nghi"
+    
+    line,4,some\rjunk
+    line,5,jkl
+    END_DATA
+    lines = bad_data.to_a
+    assert_equal(6, lines.size)
+    assert_match(/\Aline,4/, lines.find { |l| l =~ /some\rjunk/ })
+    
+    csv = FasterCSV.new(bad_data)
+    begin
+      loop do
+        assert_not_nil(csv.shift)
+        assert_send([csv.lineno, :<, 4])
+      end
+    rescue FasterCSV::MalformedCSVError
+      assert_equal( "Unquoted fields do not allow \\r or \\n (line 4).",
+                    $!.message )
+    end
+
+    assert_nothing_raised(FasterCSV::MalformedCSVError) do 
+      FasterCSV.parse_line('1,2,"3...',{:raise_exception => false})
+    end
+    assert_raise(FasterCSV::MalformedCSVError) do 
+      FasterCSV.parse_line('1,2,"3...')
+    end
+    
+    bad_data = <<-END_DATA.gsub(/^ +/, "")
+    line,1,abc
+    line,2,"def\nghi"
+    
+    line,4,8'10"
+    line,5,jkl
+    END_DATA
+    lines = bad_data.to_a
+    assert_equal(6, lines.size)
+    assert_match(/\Aline,4/, lines.find { |l| l =~ /8'10"/ })
+    
+    csv = FasterCSV.new(bad_data)
+    begin
+      loop do
+        assert_not_nil(csv.shift)
+        assert_send([csv.lineno, :<, 4])
+      end
+    rescue FasterCSV::MalformedCSVError
+      assert_equal("Illegal quoting on line 4.", $!.message)
+    end
+  end
+end

data/test/tc_csv_writing.rb

@@ -0,0 +1,96 @@
+#!/usr/local/bin/ruby -w
+
+# tc_csv_writing.rb
+#
+#  Created by James Edward Gray II on 2005-11-14.
+#  Copyright 2005 Gray Productions. All rights reserved.
+
+require "test/unit"
+
+require "faster_csv"
+
+class TestFasterCSVWriting < Test::Unit::TestCase
+  def test_writing
+    [ ["\t",                      ["\t"]],
+      ["foo,\"\"\"\"\"\",baz",    ["foo", "\"\"", "baz"]],
+      ["foo,\"\"\"bar\"\"\",baz", ["foo", "\"bar\"", "baz"]],
+      ["\"\"\"\n\",\"\"\"\n\"",   ["\"\n", "\"\n"]],
+      ["foo,\"\r\n\",baz",        ["foo", "\r\n", "baz"]],
+      ["\"\"",                    [""]],
+      ["foo,\"\"\"\",baz",        ["foo", "\"", "baz"]],
+      ["foo,\"\r.\n\",baz",       ["foo", "\r.\n", "baz"]],
+      ["foo,\"\r\",baz",          ["foo", "\r", "baz"]],
+      ["foo,\"\",baz",            ["foo", "", "baz"]],
+      ["\",\"",                   [","]],
+      ["foo",                     ["foo"]],
+      [",,",                      [nil, nil, nil]],
+      [",",                       [nil, nil]],
+      ["foo,\"\n\",baz",          ["foo", "\n", "baz"]],
+      ["foo,,baz",                ["foo", nil, "baz"]],
+      ["\"\"\"\r\",\"\"\"\r\"",   ["\"\r", "\"\r"]],
+      ["\",\",\",\"",             [",", ","]],
+      ["foo,bar,",                ["foo", "bar", nil]],
+      [",foo,bar",                [nil, "foo", "bar"]],
+      ["foo,bar",                 ["foo", "bar"]],
+      [";",                       [";"]],
+      ["\t,\t",                   ["\t", "\t"]],
+      ["foo,\"\r\n\r\",baz",      ["foo", "\r\n\r", "baz"]],
+      ["foo,\"\r\n\n\",baz",      ["foo", "\r\n\n", "baz"]],
+      ["foo,\"foo,bar\",baz",     ["foo", "foo,bar", "baz"]],
+      [";,;",                     [";", ";"]],
+      ["foo,\"\"\"\"\"\",baz",    ["foo", "\"\"", "baz"]],
+      ["foo,\"\"\"bar\"\"\",baz", ["foo", "\"bar\"", "baz"]],
+      ["foo,\"\r\n\",baz",        ["foo", "\r\n", "baz"]],
+      ["\"\"",                    [""]],
+      ["foo,\"\"\"\",baz",        ["foo", "\"", "baz"]],
+      ["foo,\"\r.\n\",baz",       ["foo", "\r.\n", "baz"]],
+      ["foo,\"\r\",baz",          ["foo", "\r", "baz"]],
+      ["foo,\"\",baz",            ["foo", "", "baz"]],
+      ["foo",                     ["foo"]],
+      [",,",                      [nil, nil, nil]],
+      [",",                       [nil, nil]],
+      ["foo,\"\n\",baz",          ["foo", "\n", "baz"]],
+      ["foo,,baz",                ["foo", nil, "baz"]],
+      ["foo,bar",                 ["foo", "bar"]],
+      ["foo,\"\r\n\n\",baz",      ["foo", "\r\n\n", "baz"]],
+      ["foo,\"foo,bar\",baz",     ["foo", "foo,bar", "baz"]],
+      [%Q{a,b},                   ["a", "b"]],
+      [%Q{a,"""b"""},             ["a", "\"b\""]],
+      [%Q{a,"""b"},               ["a", "\"b"]],
+      [%Q{a,"b"""},               ["a", "b\""]],
+      [%Q{a,"\nb"""},             ["a", "\nb\""]],
+      [%Q{a,"""\nb"},             ["a", "\"\nb"]],
+      [%Q{a,"""\nb\n"""},         ["a", "\"\nb\n\""]],
+      [%Q{a,"""\nb\n""",},        ["a", "\"\nb\n\"", nil]],
+      [%Q{a,,,},                  ["a", nil, nil, nil]],
+      [%Q{,},                     [nil, nil]],
+      [%Q{"",""},                 ["", ""]],
+      [%Q{""""},                  ["\""]],
+      [%Q{"""",""},               ["\"",""]],
+      [%Q{,""},                   [nil,""]],
+      [%Q{,"\r"},                 [nil,"\r"]],
+      [%Q{"\r\n,"},               ["\r\n,"]],
+      [%Q{"\r\n,",},              ["\r\n,", nil]] ].each do |test_case|
+        assert_equal( test_case.first + $/,
+                      FasterCSV.generate_line(test_case.last) )
+      end
+  end
+  
+  def test_col_sep
+    assert_equal( "a;b;;c\n", FasterCSV.generate_line( ["a", "b", nil, "c"],
+                                                       :col_sep => ";" ) )
+    assert_equal( "a\tb\t\tc\n", FasterCSV.generate_line( ["a", "b", nil, "c"],
+                                                          :col_sep => "\t" ) )
+  end
+  
+  def test_row_sep
+    assert_equal( "a,b,,c\r\n", FasterCSV.generate_line( ["a", "b", nil, "c"],
+                                                         :row_sep => "\r\n" ) )
+  end
+  
+  def test_force_quotes
+    assert_equal( %Q{"1","b","","already ""quoted"""\n},
+                  FasterCSV.generate_line( [1, "b", nil, %Q{already "quoted"}],
+                                           :force_quotes => true ) )
+  end
+end

data/test/tc_data_converters.rb

@@ -0,0 +1,260 @@
+#!/usr/local/bin/ruby -w
+
+# tc_data_converters.rb
+#
+#  Created by James Edward Gray II on 2005-12-30.
+#  Copyright 2005 Gray Productions. All rights reserved.
+
+require "test/unit"
+
+require "faster_csv"
+
+class TestDataConverters < Test::Unit::TestCase
+  def setup
+    @data   = "Numbers,:integer,1,:float,3.015"
+    @parser = FasterCSV.new(@data)
+    
+    @custom = lambda { |field| field =~ /\A:(\S.*?)\s*\Z/ ? $1.to_sym : field }
+    
+    @win_safe_time_str = Time.now.strftime("%a %b %d %H:%M:%S %Y")
+  end
+  
+  def test_builtin_integer_converter
+    # does convert
+    [-5, 1, 10000000000].each do |n|
+      assert_equal(n, FasterCSV::Converters[:integer][n.to_s])
+    end
+    
+    # does not convert
+    (%w{junk 1.0} + [""]).each do |str|
+      assert_equal(str, FasterCSV::Converters[:integer][str])
+    end
+  end
+
+  def test_builtin_float_converter
+    # does convert
+    [-5.1234, 0, 2.3e-11].each do |n|
+      assert_equal(n, FasterCSV::Converters[:float][n.to_s])
+    end
+    
+    # does not convert
+    (%w{junk 1..0 .015F} + [""]).each do |str|
+      assert_equal(str, FasterCSV::Converters[:float][str])
+    end
+  end
+
+  def test_builtin_date_converter
+    # does convert
+    assert_instance_of(
+      Date,
+      FasterCSV::Converters[:date][@win_safe_time_str.sub(/\d+:\d+:\d+ /, "")]
+    )
+
+    # does not convert
+    assert_instance_of(String, FasterCSV::Converters[:date]["junk"])
+  end
+
+  def test_builtin_date_time_converter
+    # does convert
+    assert_instance_of( DateTime,
+                        FasterCSV::Converters[:date_time][@win_safe_time_str] )
+
+    # does not convert
+    assert_instance_of(String, FasterCSV::Converters[:date_time]["junk"])
+  end
+  
+  def test_convert_with_builtin
+    # setup parser...
+    assert(@parser.respond_to?(:convert))
+    assert_nothing_raised(Exception) { @parser.convert(:integer) }
+    
+    # and use
+    assert_equal(["Numbers", ":integer", 1, ":float", "3.015"], @parser.shift)
+    
+    setup  # reset
+    
+    # setup parser...
+    assert_nothing_raised(Exception) { @parser.convert(:float) }
+    
+    # and use
+    assert_equal(["Numbers", ":integer", 1.0, ":float", 3.015], @parser.shift)
+  end
+  
+  def test_convert_order
+    # floats first, then integers...
+    assert_nothing_raised(Exception) do 
+      @parser.convert(:float)
+      @parser.convert(:integer)
+    end
+    
+    # gets us nothing but floats
+    assert_equal( [String, String, Float, String, Float],
+                  @parser.shift.map { |field| field.class } )
+    
+    setup  # reset
+
+    # integers have precendance...
+    assert_nothing_raised(Exception) do 
+      @parser.convert(:integer)
+      @parser.convert(:float)
+    end
+    
+    # gives us proper number conversion
+    assert_equal( [String, String, Fixnum, String, Float],
+                  @parser.shift.map { |field| field.class } )
+  end
+
+  def test_builtin_numeric_combo_converter
+    # setup parser...
+    assert_nothing_raised(Exception) { @parser.convert(:numeric) }
+    
+    # and use
+    assert_equal( [String, String, Fixnum, String, Float],
+                  @parser.shift.map { |field| field.class } )
+  end
+
+  def test_builtin_all_nested_combo_converter
+    # setup parser...
+    @data   << ",#{@win_safe_time_str}"        # add a DateTime field
+    @parser =  FasterCSV.new(@data)            # reset parser
+    assert_nothing_raised(Exception) { @parser.convert(:all) }
+    
+    # and use
+    assert_equal( [String, String, Fixnum, String, Float, DateTime],
+                  @parser.shift.map { |field| field.class } )
+  end
+  
+  def test_convert_with_custom_code
+    # define custom converter...
+    assert_nothing_raised(Exception) do
+      @parser.convert { |field| field =~ /\A:(\S.*?)\s*\Z/ ? $1.to_sym : field }
+    end
+    
+    # and use
+    assert_equal(["Numbers", :integer, "1", :float, "3.015"], @parser.shift)
+
+    setup  # reset
+
+    # mix built-in and custom...
+    assert_nothing_raised(Exception) { @parser.convert(:numeric) }
+    assert_nothing_raised(Exception) { @parser.convert(&@custom) }
+    
+    # and use
+    assert_equal(["Numbers", :integer, 1, :float, 3.015], @parser.shift)
+  end
+  
+  def test_convert_with_custom_code_using_field_info
+    # define custom converter that uses field information...
+    assert_nothing_raised(Exception) do
+      @parser.convert do |field, info|
+        assert_equal(1, info.line)
+        info.index == 4 ? Float(field).floor : field
+      end
+    end
+    
+    # and use
+    assert_equal(["Numbers", ":integer", "1", ":float", 3], @parser.shift)
+  end
+  
+  def test_convert_with_custom_code_using_field_info_header
+    @parser = FasterCSV.new(@data, :headers => %w{one two three four five})
+    
+    # define custom converter that uses field header information...
+    assert_nothing_raised(Exception) do
+      @parser.convert do |field, info|
+        info.header == "three" ? Integer(field) * 100 : field
+      end
+    end
+    
+    # and use
+    assert_equal(["Numbers", ":integer", 100, ":float", "3.015"], @parser.shift.fields)
+  end
+  
+  def test_shortcut_interface
+    assert_equal( ["Numbers", ":integer", 1, ":float", 3.015],
+                  FasterCSV.parse_line(@data, :converters => :numeric) )
+
+    assert_equal( ["Numbers", ":integer", 1, ":float", 3.015],
+                  FasterCSV.parse_line( @data, :converters => [ :integer,
+                                                                :float ] ) )
+
+    assert_equal( ["Numbers", :integer, 1, :float, 3.015],
+                  FasterCSV.parse_line( @data, :converters => [ :numeric,
+                                                                @custom ] ) )
+  end
+  
+  def test_unconverted_fields
+    [ [ @data,
+        ["Numbers", :integer, 1, :float, 3.015],
+        %w{Numbers :integer 1 :float 3.015} ],
+      ["\n", Array.new, Array.new] ].each do |test, fields, unconverted|
+      row = nil
+      assert_nothing_raised(Exception) do 
+        row = FasterCSV.parse_line( test,
+                                    :converters         => [:numeric, @custom],
+                                    :unconverted_fields => true )
+      end
+      assert_not_nil(row)
+      assert_equal(fields, row)
+      assert_respond_to(row, :unconverted_fields)
+      assert_equal(unconverted, row.unconverted_fields)
+    end
+
+    data = <<-END_CSV.gsub(/^\s+/, "")
+    first,second,third
+    1,2,3
+    END_CSV
+    row = nil
+    assert_nothing_raised(Exception) do 
+      row = FasterCSV.parse_line( data,
+                                  :converters         => :numeric,
+                                  :unconverted_fields => true,
+                                  :headers            => :first_row )
+    end
+    assert_not_nil(row)
+    assert_equal([["first", 1], ["second", 2], ["third", 3]], row.to_a)
+    assert_respond_to(row, :unconverted_fields)
+    assert_equal(%w{1 2 3}, row.unconverted_fields)
+
+    assert_nothing_raised(Exception) do 
+      row = FasterCSV.parse_line( data,
+                                  :converters         => :numeric,
+                                  :unconverted_fields => true,
+                                  :headers            => :first_row,
+                                  :return_headers     => true )
+    end
+    assert_not_nil(row)
+    assert_equal( [%w{first first}, %w{second second}, %w{third third}],
+                  row.to_a )
+    assert_respond_to(row, :unconverted_fields)
+    assert_equal(%w{first second third}, row.unconverted_fields)
+
+    assert_nothing_raised(Exception) do 
+      row = FasterCSV.parse_line( data,
+                                  :converters         => :numeric,
+                                  :unconverted_fields => true,
+                                  :headers            => :first_row,
+                                  :return_headers     => true,
+                                  :header_converters  => :symbol )
+    end
+    assert_not_nil(row)
+    assert_equal( [[:first, "first"], [:second, "second"], [:third, "third"]],
+                  row.to_a )
+    assert_respond_to(row, :unconverted_fields)
+    assert_equal(%w{first second third}, row.unconverted_fields)
+
+    assert_nothing_raised(Exception) do 
+      row = FasterCSV.parse_line( data,
+                                  :converters         => :numeric,
+                                  :unconverted_fields => true,
+                                  :headers            => %w{my new headers},
+                                  :return_headers     => true,
+                                  :header_converters  => :symbol )
+    end
+    assert_not_nil(row)
+    assert_equal( [[:my, "my"], [:new, "new"], [:headers, "headers"]],
+                  row.to_a )
+    assert_respond_to(row, :unconverted_fields)
+    assert_equal(Array.new, row.unconverted_fields)
+  end
+end