fastcsv 0.0.2 → 0.0.3

data/test/csv/test_csv_writing.rb

@@ -0,0 +1,97 @@
+#!/usr/bin/env ruby -w
+# encoding: UTF-8
+
+# tc_csv_writing.rb
+#
+#  Created by James Edward Gray II on 2005-10-31.
+#  Copyright 2005 James Edward Gray II. You can redistribute or modify this code
+#  under the terms of Ruby's license.
+
+require_relative "base"
+
+class TestCSV::Writing < TestCSV
+  extend DifferentOFS
+
+  def test_writing
+    [ ["\t",                      ["\t"]],
+      ["foo,\"\"\"\"\"\",baz",    ["foo", "\"\"", "baz"]],
+      ["foo,\"\"\"bar\"\"\",baz", ["foo", "\"bar\"", "baz"]],
+      ["\"\"\"\n\",\"\"\"\n\"",   ["\"\n", "\"\n"]],
+      ["foo,\"\r\n\",baz",        ["foo", "\r\n", "baz"]],
+      ["\"\"",                    [""]],
+      ["foo,\"\"\"\",baz",        ["foo", "\"", "baz"]],
+      ["foo,\"\r.\n\",baz",       ["foo", "\r.\n", "baz"]],
+      ["foo,\"\r\",baz",          ["foo", "\r", "baz"]],
+      ["foo,\"\",baz",            ["foo", "", "baz"]],
+      ["\",\"",                   [","]],
+      ["foo",                     ["foo"]],
+      [",,",                      [nil, nil, nil]],
+      [",",                       [nil, nil]],
+      ["foo,\"\n\",baz",          ["foo", "\n", "baz"]],
+      ["foo,,baz",                ["foo", nil, "baz"]],
+      ["\"\"\"\r\",\"\"\"\r\"",   ["\"\r", "\"\r"]],
+      ["\",\",\",\"",             [",", ","]],
+      ["foo,bar,",                ["foo", "bar", nil]],
+      [",foo,bar",                [nil, "foo", "bar"]],
+      ["foo,bar",                 ["foo", "bar"]],
+      [";",                       [";"]],
+      ["\t,\t",                   ["\t", "\t"]],
+      ["foo,\"\r\n\r\",baz",      ["foo", "\r\n\r", "baz"]],
+      ["foo,\"\r\n\n\",baz",      ["foo", "\r\n\n", "baz"]],
+      ["foo,\"foo,bar\",baz",     ["foo", "foo,bar", "baz"]],
+      [";,;",                     [";", ";"]],
+      ["foo,\"\"\"\"\"\",baz",    ["foo", "\"\"", "baz"]],
+      ["foo,\"\"\"bar\"\"\",baz", ["foo", "\"bar\"", "baz"]],
+      ["foo,\"\r\n\",baz",        ["foo", "\r\n", "baz"]],
+      ["\"\"",                    [""]],
+      ["foo,\"\"\"\",baz",        ["foo", "\"", "baz"]],
+      ["foo,\"\r.\n\",baz",       ["foo", "\r.\n", "baz"]],
+      ["foo,\"\r\",baz",          ["foo", "\r", "baz"]],
+      ["foo,\"\",baz",            ["foo", "", "baz"]],
+      ["foo",                     ["foo"]],
+      [",,",                      [nil, nil, nil]],
+      [",",                       [nil, nil]],
+      ["foo,\"\n\",baz",          ["foo", "\n", "baz"]],
+      ["foo,,baz",                ["foo", nil, "baz"]],
+      ["foo,bar",                 ["foo", "bar"]],
+      ["foo,\"\r\n\n\",baz",      ["foo", "\r\n\n", "baz"]],
+      ["foo,\"foo,bar\",baz",     ["foo", "foo,bar", "baz"]],
+      [%Q{a,b},                   ["a", "b"]],
+      [%Q{a,"""b"""},             ["a", "\"b\""]],
+      [%Q{a,"""b"},               ["a", "\"b"]],
+      [%Q{a,"b"""},               ["a", "b\""]],
+      [%Q{a,"\nb"""},             ["a", "\nb\""]],
+      [%Q{a,"""\nb"},             ["a", "\"\nb"]],
+      [%Q{a,"""\nb\n"""},         ["a", "\"\nb\n\""]],
+      [%Q{a,"""\nb\n""",},        ["a", "\"\nb\n\"", nil]],
+      [%Q{a,,,},                  ["a", nil, nil, nil]],
+      [%Q{,},                     [nil, nil]],
+      [%Q{"",""},                 ["", ""]],
+      [%Q{""""},                  ["\""]],
+      [%Q{"""",""},               ["\"",""]],
+      [%Q{,""},                   [nil,""]],
+      [%Q{,"\r"},                 [nil,"\r"]],
+      [%Q{"\r\n,"},               ["\r\n,"]],
+      [%Q{"\r\n,",},              ["\r\n,", nil]] ].each do |test_case|
+        assert_equal(test_case.first + $/, FastCSV.generate_line(test_case.last))
+      end
+  end
+
+  def test_col_sep
+    assert_equal( "a;b;;c\n", FastCSV.generate_line( ["a", "b", nil, "c"],
+                                                 col_sep: ";" ) )
+    assert_equal( "a\tb\t\tc\n", FastCSV.generate_line( ["a", "b", nil, "c"],
+                                                    col_sep: "\t" ) )
+  end
+
+  def test_row_sep
+    assert_equal( "a,b,,c\r\n", FastCSV.generate_line( ["a", "b", nil, "c"],
+                                                   row_sep: "\r\n" ) )
+  end
+
+  def test_force_quotes
+    assert_equal( %Q{"1","b","","already ""quoted"""\n},
+                  FastCSV.generate_line( [1, "b", nil, %Q{already "quoted"}],
+                                     force_quotes: true ) )
+  end
+end

data/test/csv/test_data_converters.rb

@@ -0,0 +1,263 @@
+#!/usr/bin/env ruby -w
+# encoding: UTF-8
+
+# tc_data_converters.rb
+#
+#  Created by James Edward Gray II on 2005-10-31.
+#  Copyright 2005 James Edward Gray II. You can redistribute or modify this code
+#  under the terms of Ruby's license.
+
+require_relative "base"
+
+class TestCSV::DataConverters < TestCSV
+  extend DifferentOFS
+
+  def setup
+    super
+    @data   = "Numbers,:integer,1,:float,3.015"
+    @parser = FastCSV.new(@data)
+
+    @custom = lambda { |field| field =~ /\A:(\S.*?)\s*\Z/ ? $1.to_sym : field }
+
+    @win_safe_time_str = Time.now.strftime("%a %b %d %H:%M:%S %Y")
+  end
+
+  def test_builtin_integer_converter
+    # does convert
+    [-5, 1, 10000000000].each do |n|
+      assert_equal(n, FastCSV::Converters[:integer][n.to_s])
+    end
+
+    # does not convert
+    (%w{junk 1.0} + [""]).each do |str|
+      assert_equal(str, FastCSV::Converters[:integer][str])
+    end
+  end
+
+  def test_builtin_float_converter
+    # does convert
+    [-5.1234, 0, 2.3e-11].each do |n|
+      assert_equal(n, FastCSV::Converters[:float][n.to_s])
+    end
+
+    # does not convert
+    (%w{junk 1..0 .015F} + [""]).each do |str|
+      assert_equal(str, FastCSV::Converters[:float][str])
+    end
+  end
+
+  def test_builtin_date_converter
+    # does convert
+    assert_instance_of(
+      Date,
+      FastCSV::Converters[:date][@win_safe_time_str.sub(/\d+:\d+:\d+ /, "")]
+    )
+
+    # does not convert
+    assert_instance_of(String, FastCSV::Converters[:date]["junk"])
+  end
+
+  def test_builtin_date_time_converter
+    # does convert
+    assert_instance_of( DateTime,
+                        FastCSV::Converters[:date_time][@win_safe_time_str] )
+
+    # does not convert
+    assert_instance_of(String, FastCSV::Converters[:date_time]["junk"])
+  end
+
+  def test_convert_with_builtin_integer
+    # setup parser...
+    assert(@parser.respond_to?(:convert))
+    assert_nothing_raised(Exception) { @parser.convert(:integer) }
+
+    # and use
+    assert_equal(["Numbers", ":integer", 1, ":float", "3.015"], @parser.shift)
+  end
+
+  def test_convert_with_builtin_float
+    # setup parser...
+    assert(@parser.respond_to?(:convert))
+    assert_nothing_raised(Exception) { @parser.convert(:float) }
+
+    # and use
+    assert_equal(["Numbers", ":integer", 1.0, ":float", 3.015], @parser.shift)
+  end
+
+  def test_convert_order_float_integer
+    # floats first, then integers...
+    assert_nothing_raised(Exception) do
+      @parser.convert(:float)
+      @parser.convert(:integer)
+    end
+
+    # gets us nothing but floats
+    assert_equal( [String, String, Float, String, Float],
+                  @parser.shift.map { |field| field.class } )
+  end
+
+  def test_convert_order_integer_float
+    # integers have precendance...
+    assert_nothing_raised(Exception) do
+      @parser.convert(:integer)
+      @parser.convert(:float)
+    end
+
+    # gives us proper number conversion
+    assert_equal( [String, String, Fixnum, String, Float],
+                  @parser.shift.map { |field| field.class } )
+  end
+
+  def test_builtin_numeric_combo_converter
+    # setup parser...
+    assert_nothing_raised(Exception) { @parser.convert(:numeric) }
+
+    # and use
+    assert_equal( [String, String, Fixnum, String, Float],
+                  @parser.shift.map { |field| field.class } )
+  end
+
+  def test_builtin_all_nested_combo_converter
+    # setup parser...
+    @data   << ",#{@win_safe_time_str}"        # add a DateTime field
+    @parser =  FastCSV.new(@data)                  # reset parser
+    assert_nothing_raised(Exception) { @parser.convert(:all) }
+
+    # and use
+    assert_equal( [String, String, Fixnum, String, Float, DateTime],
+                  @parser.shift.map { |field| field.class } )
+  end
+
+  def test_convert_with_custom_code
+    # define custom converter...
+    assert_nothing_raised(Exception) do
+      @parser.convert { |field| field =~ /\A:(\S.*?)\s*\Z/ ? $1.to_sym : field }
+    end
+
+    # and use
+    assert_equal(["Numbers", :integer, "1", :float, "3.015"], @parser.shift)
+  end
+
+  def test_convert_with_custom_code_mix
+    # mix built-in and custom...
+    assert_nothing_raised(Exception) { @parser.convert(:numeric) }
+    assert_nothing_raised(Exception) { @parser.convert(&@custom) }
+
+    # and use
+    assert_equal(["Numbers", :integer, 1, :float, 3.015], @parser.shift)
+  end
+
+  def test_convert_with_custom_code_using_field_info
+    # define custom converter that uses field information...
+    assert_nothing_raised(Exception) do
+      @parser.convert do |field, info|
+        assert_equal(1, info.line)
+        info.index == 4 ? Float(field).floor : field
+      end
+    end
+
+    # and use
+    assert_equal(["Numbers", ":integer", "1", ":float", 3], @parser.shift)
+  end
+
+  def test_convert_with_custom_code_using_field_info_header
+    @parser = FastCSV.new(@data, headers: %w{one two three four five})
+
+    # define custom converter that uses field header information...
+    assert_nothing_raised(Exception) do
+      @parser.convert do |field, info|
+        info.header == "three" ? Integer(field) * 100 : field
+      end
+    end
+
+    # and use
+    assert_equal( ["Numbers", ":integer", 100, ":float", "3.015"],
+                  @parser.shift.fields )
+  end
+
+  def test_shortcut_interface
+    assert_equal( ["Numbers", ":integer", 1, ":float", 3.015],
+                  FastCSV.parse_line(@data, converters: :numeric) )
+
+    assert_equal( ["Numbers", ":integer", 1, ":float", 3.015],
+                  FastCSV.parse_line(@data, converters: [:integer, :float]) )
+
+    assert_equal( ["Numbers", :integer, 1, :float, 3.015],
+                  FastCSV.parse_line(@data, converters: [:numeric, @custom]) )
+  end
+
+  def test_unconverted_fields
+    [ [ @data,
+        ["Numbers", :integer, 1, :float, 3.015],
+        %w{Numbers :integer 1 :float 3.015} ],
+      ["\n", Array.new, Array.new] ].each do |test, fields, unconverted|
+      row = nil
+      assert_nothing_raised(Exception) do
+        row = FastCSV.parse_line( test,
+                              converters:         [:numeric, @custom],
+                              unconverted_fields: true )
+      end
+      assert_not_nil(row)
+      assert_equal(fields, row)
+      assert_respond_to(row, :unconverted_fields)
+      assert_equal(unconverted, row.unconverted_fields)
+    end
+
+    data = <<-END_CSV.gsub(/^\s+/, "")
+    first,second,third
+    1,2,3
+    END_CSV
+    row = nil
+    assert_nothing_raised(Exception) do
+      row = FastCSV.parse_line( data,
+                            converters:         :numeric,
+                            unconverted_fields: true,
+                            headers:            :first_row )
+    end
+    assert_not_nil(row)
+    assert_equal([["first", 1], ["second", 2], ["third", 3]], row.to_a)
+    assert_respond_to(row, :unconverted_fields)
+    assert_equal(%w{1 2 3}, row.unconverted_fields)
+
+    assert_nothing_raised(Exception) do
+      row = FastCSV.parse_line( data,
+                            converters:         :numeric,
+                            unconverted_fields: true,
+                            headers:            :first_row,
+                            return_headers:     true )
+    end
+    assert_not_nil(row)
+    assert_equal( [%w{first first}, %w{second second}, %w{third third}],
+                  row.to_a )
+    assert_respond_to(row, :unconverted_fields)
+    assert_equal(%w{first second third}, row.unconverted_fields)
+
+    assert_nothing_raised(Exception) do
+      row = FastCSV.parse_line( data,
+                            converters:         :numeric,
+                            unconverted_fields: true,
+                            headers:            :first_row,
+                            return_headers:     true,
+                            header_converters:  :symbol )
+    end
+    assert_not_nil(row)
+    assert_equal( [[:first, "first"], [:second, "second"], [:third, "third"]],
+                  row.to_a )
+    assert_respond_to(row, :unconverted_fields)
+    assert_equal(%w{first second third}, row.unconverted_fields)
+
+    assert_nothing_raised(Exception) do
+      row = FastCSV.parse_line( data,
+                            converters:         :numeric,
+                            unconverted_fields: true,
+                            headers:            %w{my new headers},
+                            return_headers:     true,
+                            header_converters:  :symbol )
+    end
+    assert_not_nil(row)
+    assert_equal( [[:my, "my"], [:new, "new"], [:headers, "headers"]],
+                  row.to_a )
+    assert_respond_to(row, :unconverted_fields)
+    assert_equal(Array.new, row.unconverted_fields)
+  end
+end

data/test/csv/test_encodings.rb

@@ -0,0 +1,339 @@
+#!/usr/bin/env ruby -w
+# encoding: UTF-8
+
+# tc_encodings.rb
+#
+#  Created by James Edward Gray II on 2008-09-13.
+#  Copyright 2008 James Edward Gray II. You can redistribute or modify this code
+#  under the terms of Ruby's license.
+
+require_relative "base"
+
+class TestCSV::Encodings < TestCSV
+  extend DifferentOFS
+
+  def setup
+    super
+    require 'tempfile'
+    @temp_csv_file = Tempfile.new(%w"test_csv. .csv")
+    @temp_csv_path = @temp_csv_file.path
+    @temp_csv_file.close
+  end
+
+  def teardown
+    @temp_csv_file.close!
+    super
+  end
+
+  ########################################
+  ### Hand Test Some Popular Encodings ###
+  ########################################
+
+  def test_parses_utf8_encoding
+    assert_parses( [ %w[ one two … ],
+                     %w[ 1   …   3 ],
+                     %w[ …   5   6 ] ], "UTF-8" )
+  end
+
+  def test_parses_latin1_encoding
+    assert_parses( [ %w[ one    two    Résumé ],
+                     %w[ 1      Résumé 3      ],
+                     %w[ Résumé 5      6      ] ], "ISO-8859-1" )
+  end
+
+  # def test_parses_utf16be_encoding
+  #   assert_parses( [ %w[ one two … ],
+  #                    %w[ 1   …   3 ],
+  #                    %w[ …   5   6 ] ], "UTF-16BE" )
+  # end
+
+  def test_parses_shift_jis_encoding
+    assert_parses( [ %w[ 一 二 三 ],
+                     %w[ 四 五 六 ],
+                     %w[ 七 八 九 ] ], "Shift_JIS" )
+  end
+
+  ###########################################################
+  ### Try Simple Reading for All Non-dummy Ruby Encodings ###
+  ###########################################################
+
+  def test_reading_with_most_encodings
+    each_encoding do |encoding|
+      begin
+        assert_parses( [ %w[ abc def ],
+                         %w[ ghi jkl ] ], encoding )
+      rescue Encoding::ConverterNotFoundError
+        fail("Failed to support #{encoding.name}.")
+      end
+    end
+  end
+
+  def test_regular_expression_escaping
+    each_encoding do |encoding|
+      begin
+        assert_parses( [ %w[ abc def ],
+                         %w[ ghi jkl ] ], encoding, col_sep: "," )
+      rescue Encoding::ConverterNotFoundError
+        fail("Failed to properly escape #{encoding.name}.")
+      end
+    end
+  end
+
+  def test_read_with_default_encoding
+    data             = "abc"
+    default_external = Encoding.default_external
+    each_encoding do |encoding|
+      File.open(@temp_csv_path, "wb", encoding: encoding) {|f| f << data}
+      begin
+        no_warnings do
+          Encoding.default_external = encoding
+        end
+        result = FastCSV.read(@temp_csv_path)[0][0]
+      ensure
+        no_warnings do
+          Encoding.default_external = default_external
+        end
+      end
+      assert_equal(encoding, result.encoding)
+    end
+  end
+
+  #######################################################################
+  ### Stress Test ASCII Compatible and Non-ASCII Compatible Encodings ###
+  #######################################################################
+
+  def test_auto_line_ending_detection
+    # arrange data to place a \r at the end of FastCSV's read ahead point
+    encode_for_tests([["a" * 509]], row_sep: "\r\n") do |data|
+      assert_equal("\r\n".encode(data.encoding), FastCSV.new(data).row_sep)
+    end
+  end
+
+  def test_csv_chars_are_transcoded
+    encode_for_tests([%w[abc def]]) do |data|
+      %w[col_sep row_sep quote_char].each do |csv_char|
+        assert_equal( ",".encode(data.encoding),
+                      FastCSV.new(data, csv_char.to_sym => ",").send(csv_char) )
+      end
+    end
+  end
+
+  def test_parser_works_with_encoded_headers
+    encode_for_tests([%w[one two three], %w[1 2 3]]) do |data|
+      parsed = FastCSV.parse(data, headers: true)
+      assert( parsed.headers.all? { |h| h.encoding == data.encoding },
+              "Wrong data encoding." )
+      parsed.each do |row|
+        assert( row.fields.all? { |f| f.encoding == data.encoding },
+                "Wrong data encoding." )
+      end
+    end
+  end
+
+  def test_built_in_converters_transcode_to_utf_8_then_convert
+    encode_for_tests([%w[one two three], %w[1 2 3]]) do |data|
+      parsed = FastCSV.parse(data, converters: :integer)
+      assert( parsed[0].all? { |f| f.encoding == data.encoding },
+              "Wrong data encoding." )
+      assert_equal([1, 2, 3], parsed[1])
+    end
+  end
+
+  def test_built_in_header_converters_transcode_to_utf_8_then_convert
+    encode_for_tests([%w[one two three], %w[1 2 3]]) do |data|
+      parsed = FastCSV.parse( data, headers:           true,
+                                header_converters: :downcase )
+      assert( parsed.headers.all? { |h| h.encoding.name == "UTF-8" },
+              "Wrong data encoding." )
+      assert( parsed[0].fields.all? { |f| f.encoding == data.encoding },
+              "Wrong data encoding." )
+    end
+  end
+
+  def test_open_allows_you_to_set_encodings
+    encode_for_tests([%w[abc def]]) do |data|
+      # read and write in encoding
+      File.open(@temp_csv_path, "wb:#{data.encoding.name}") { |f| f << data }
+      FastCSV.open(@temp_csv_path, "rb:#{data.encoding.name}") do |csv|
+        csv.each do |row|
+          assert( row.all? { |f| f.encoding == data.encoding },
+                  "Wrong data encoding." )
+        end
+      end
+
+      # read and write with transcoding
+      # File.open(@temp_csv_path, "wb:UTF-32BE:#{data.encoding.name}") do |f|
+      #   f << data
+      # end
+      # FastCSV.open(@temp_csv_path, "rb:UTF-32BE:#{data.encoding.name}") do |csv|
+      #   csv.each do |row|
+      #     assert( row.all? { |f| f.encoding == data.encoding },
+      #             "Wrong data encoding." )
+      #   end
+      # end
+    end
+  end
+
+  def test_foreach_allows_you_to_set_encodings
+    encode_for_tests([%w[abc def]]) do |data|
+      # read and write in encoding
+      File.open(@temp_csv_path, "wb", encoding: data.encoding) { |f| f << data }
+      FastCSV.foreach(@temp_csv_path, encoding: data.encoding) do |row|
+        row.each {|f| assert_equal(f.encoding, data.encoding)}
+      end
+
+      # read and write with transcoding
+      # File.open(@temp_csv_path, "wb:UTF-32BE:#{data.encoding.name}") do |f|
+      #   f << data
+      # end
+      # FastCSV.foreach( @temp_csv_path,
+      #              encoding: "UTF-32BE:#{data.encoding.name}" ) do |row|
+      #   assert( row.all? { |f| f.encoding == data.encoding },
+      #           "Wrong data encoding." )
+      # end
+    end
+  end
+
+  def test_read_allows_you_to_set_encodings
+    encode_for_tests([%w[abc def]]) do |data|
+      # read and write in encoding
+      File.open(@temp_csv_path, "wb:#{data.encoding.name}") { |f| f << data }
+      rows = FastCSV.read(@temp_csv_path, encoding: data.encoding.name)
+      assert( rows.flatten.all? { |f| f.encoding == data.encoding },
+              "Wrong data encoding." )
+
+      # read and write with transcoding
+      # File.open(@temp_csv_path, "wb:UTF-32BE:#{data.encoding.name}") do |f|
+      #   f << data
+      # end
+      # rows = FastCSV.read( @temp_csv_path,
+      #                  encoding: "UTF-32BE:#{data.encoding.name}" )
+      # assert( rows.flatten.all? { |f| f.encoding == data.encoding },
+      #         "Wrong data encoding." )
+    end
+  end
+
+  #################################
+  ### Write FastCSV in any Encoding ###
+  #################################
+
+  def test_can_write_csv_in_any_encoding
+    each_encoding do |encoding|
+      # test generate_line with encoding hint
+      begin
+        csv = %w[abc d,ef].map { |f| f.encode(encoding) }.
+          to_csv(col_sep: ",", encoding: encoding.name)
+      rescue Encoding::ConverterNotFoundError
+        next
+      end
+      assert_equal(encoding, csv.encoding)
+
+      # test generate_line with encoding guessing from fields
+      csv = %w[abc d,ef].map { |f| f.encode(encoding) }.to_csv(col_sep: ",")
+      assert_equal(encoding, csv.encoding)
+
+      # writing to files
+      data = encode_ary([%w[abc d,ef], %w[123 456 ]], encoding)
+      FastCSV.open(@temp_csv_path, "wb:#{encoding.name}") do |f|
+        data.each { |row| f << row }
+      end
+      assert_equal(data, FastCSV.read(@temp_csv_path, encoding: encoding.name))
+    end
+  end
+
+  def test_encoding_is_upgraded_during_writing_as_needed
+    data = ["foo".force_encoding("US-ASCII"), "\u3042"]
+    assert_equal("US-ASCII", data.first.encoding.name)
+    assert_equal("UTF-8",    data.last.encoding.name)
+    assert_equal("UTF-8",    data.join('').encoding.name)
+    assert_equal("UTF-8",    data.to_csv.encoding.name)
+  end
+
+  def test_encoding_is_upgraded_for_ascii_content_during_writing_as_needed
+    data = ["foo".force_encoding("ISO-8859-1"), "\u3042"]
+    assert_equal("ISO-8859-1", data.first.encoding.name)
+    assert_equal("UTF-8",      data.last.encoding.name)
+    assert_equal("UTF-8",      data.join('').encoding.name)
+    assert_equal("UTF-8",      data.to_csv.encoding.name)
+  end
+
+  private
+
+  def assert_parses(fields, encoding, options = { })
+    encoding = Encoding.find(encoding) unless encoding.is_a? Encoding
+    orig_fields = fields
+    fields   = encode_ary(fields, encoding)
+    data = ary_to_data(fields, options)
+    parsed   = FastCSV.parse(data, options)
+    assert_equal(fields, parsed)
+    parsed.flatten.each_with_index do |field, i|
+      assert_equal(encoding, field.encoding, "Field[#{i + 1}] was transcoded.")
+    end
+    File.open(@temp_csv_path, "wb") {|f| f.print(data)}
+    FastCSV.open(@temp_csv_path, "rb:#{encoding}", options) do |csv|
+      csv.each_with_index do |row, i|
+        assert_equal(fields[i], row)
+      end
+    end
+    begin
+      FastCSV.open(@temp_csv_path, "rb:#{encoding}:#{__ENCODING__}", options) do |csv|
+        csv.each_with_index do |row, i|
+          assert_equal(orig_fields[i], row)
+        end
+      end unless encoding == __ENCODING__
+    rescue Encoding::ConverterNotFoundError
+    end
+    # options[:encoding] = encoding.name
+    # FastCSV.open(@temp_csv_path, options) do |csv|
+    #   csv.each_with_index do |row, i|
+    #     assert_equal(fields[i], row)
+    #   end
+    # end
+    # options.delete(:encoding)
+    # options[:external_encoding] = encoding.name
+    # options[:internal_encoding] = __ENCODING__.name
+    # begin
+    #   FastCSV.open(@temp_csv_path, options) do |csv|
+    #     csv.each_with_index do |row, i|
+    #       assert_equal(orig_fields[i], row)
+    #     end
+    #   end unless encoding == __ENCODING__
+    # rescue Encoding::ConverterNotFoundError
+    # end
+  end
+
+  def encode_ary(ary, encoding)
+    ary.map { |row| row.map { |field| field.encode(encoding) } }
+  end
+
+  def ary_to_data(ary, options = { })
+    encoding   = ary.flatten.first.encoding
+    quote_char = (options[:quote_char] || '"').encode(encoding)
+    col_sep    = (options[:col_sep]    || ",").encode(encoding)
+    row_sep    = (options[:row_sep]    || "\n").encode(encoding)
+    ary.map { |row|
+      row.map { |field|
+        [quote_char, field.encode(encoding), quote_char].join('')
+      }.join(col_sep) + row_sep
+    }.join('').encode(encoding)
+  end
+
+  def encode_for_tests(data, options = { })
+    yield ary_to_data(encode_ary(data, "UTF-8"),    options)
+    # yield ary_to_data(encode_ary(data, "UTF-16BE"), options)
+  end
+
+  def each_encoding
+    Encoding.list.reject{|e| e.name[/\AUTF-\d\d/]}.each do |encoding|
+      next if encoding.dummy?  # skip "dummy" encodings
+      yield encoding
+    end
+  end
+
+  def no_warnings
+    old_verbose, $VERBOSE = $VERBOSE, nil
+    yield
+  ensure
+    $VERBOSE = old_verbose
+  end
+end