smarter_csv 1.3.0 → 1.5.0

data/lib/smarter_csv/smarter_csv.rb

@@ -4,48 +4,42 @@ module SmarterCSV
   class IncorrectOption < SmarterCSVException; end
   class DuplicateHeaders < SmarterCSVException; end
   class MissingHeaders < SmarterCSVException; end
-
+  class NoColSepDetected < SmarterCSVException; end
 
   def SmarterCSV.process(input, options={}, &block)   # first parameter: filename or input object with readline method
-    default_options = {:col_sep => ',' , :row_sep => $/ , :quote_char => '"', :force_simple_split => false , :verbose => false ,
-      :remove_empty_values => true, :remove_zero_values => false , :remove_values_matching => nil , :remove_empty_hashes => true , :strip_whitespace => true,
-      :convert_values_to_numeric => true, :strip_chars_from_headers => nil , :user_provided_headers => nil , :headers_in_file => true,
-      :comment_regexp => /\A#/, :chunk_size => nil , :key_mapping_hash => nil , :downcase_header => true, :strings_as_keys => false, :file_encoding => 'utf-8',
-      :remove_unmapped_keys => false, :keep_original_headers => false, :value_converters => nil, :skip_lines => nil, :force_utf8 => false, :invalid_byte_sequence => '',
-      :auto_row_sep_chars => 500, :required_headers => nil
-    }
     options = default_options.merge(options)
     options[:invalid_byte_sequence] = '' if options[:invalid_byte_sequence].nil?
-    csv_options = options.select{|k,v| [:col_sep, :row_sep, :quote_char].include?(k)} # options.slice(:col_sep, :row_sep, :quote_char)
+
     headerA = []
     result = []
-    old_row_sep = $/
     file_line_count = 0
     csv_line_count = 0
     has_rails = !! defined?(Rails)
     begin
       f = input.respond_to?(:readline) ? input : File.open(input, "r:#{options[:file_encoding]}")
 
+      # auto-detect the row separator
+      options[:row_sep] = SmarterCSV.guess_line_ending(f, options) if options[:row_sep].to_sym == :auto
+      # attempt to auto-detect column separator
+      options[:col_sep] = guess_column_separator(f, options) if options[:col_sep].to_sym == :auto
+      # preserve options, in case we need to call the CSV class
+      csv_options = options.select{|k,v| [:col_sep, :row_sep, :quote_char].include?(k)} # options.slice(:col_sep, :row_sep, :quote_char)
+      csv_options.delete(:row_sep) if [nil, :auto].include?( options[:row_sep].to_sym )
+      csv_options.delete(:col_sep) if [nil, :auto].include?( options[:col_sep].to_sym )
+
       if (options[:force_utf8] || options[:file_encoding] =~ /utf-8/i) && ( f.respond_to?(:external_encoding) && f.external_encoding != Encoding.find('UTF-8') || f.respond_to?(:encoding) && f.encoding != Encoding.find('UTF-8') )
         puts 'WARNING: you are trying to process UTF-8 input, but did not open the input with "b:utf-8" option. See README file "NOTES about File Encodings".'
       end
 
-      if options[:row_sep] == :auto
-        options[:row_sep] = line_ending = SmarterCSV.guess_line_ending( f, options )
-        f.rewind
-      end
-      $/ = options[:row_sep]
-
-      if options[:skip_lines].to_i > 0
-        options[:skip_lines].to_i.times{f.readline}
-      end
+      options[:skip_lines].to_i.times{f.readline(options[:row_sep])} if options[:skip_lines].to_i > 0
 
       if options[:headers_in_file]        # extract the header line
         # process the header line in the CSV file..
         # the first line of a CSV file contains the header .. it might be commented out, so we need to read it anyhow
-        header = f.readline
+        header = f.readline(options[:row_sep])
         header = header.force_encoding('utf-8').encode('utf-8', invalid: :replace, undef: :replace, replace: options[:invalid_byte_sequence]) if options[:force_utf8] || options[:file_encoding] !~ /utf-8/i
-        header = header.sub(options[:comment_regexp],'').chomp(options[:row_sep])
+        header = header.sub(options[:comment_regexp],'') if options[:comment_regexp]
+        header = header.chomp(options[:row_sep])
 
         file_line_count += 1
         csv_line_count += 1
@@ -60,14 +54,14 @@ module SmarterCSV
         else
           file_headerA =  header.split(options[:col_sep])
         end
+        file_header_size = file_headerA.size # before mapping, which could delete keys
+
         file_headerA.map!{|x| x.gsub(%r/#{options[:quote_char]}/,'') }
         file_headerA.map!{|x| x.strip}  if options[:strip_whitespace]
         unless options[:keep_original_headers]
           file_headerA.map!{|x| x.gsub(/\s+|-+/,'_')}
           file_headerA.map!{|x| x.downcase }   if options[:downcase_header]
         end
-
-        file_header_size = file_headerA.size
       else
         raise SmarterCSV::IncorrectOption , "ERROR: If :headers_in_file is set to false, you have to provide :user_provided_headers" if options[:user_provided_headers].nil?
       end
@@ -84,6 +78,8 @@ module SmarterCSV
       else
         headerA = file_headerA
       end
+      header_size = headerA.size # used for splitting lines
+
       headerA.map!{|x| x.to_sym } unless options[:strings_as_keys] || options[:keep_original_headers]
 
       unless options[:user_provided_headers] # wouldn't make sense to re-map user provided headers
@@ -123,7 +119,7 @@ module SmarterCSV
 
       # now on to processing all the rest of the lines in the CSV file:
       while ! f.eof?    # we can't use f.readlines() here, because this would read the whole file into memory at once, and eof => true
-        line = f.readline  # read one line.. this uses the input_record_separator $/ which we set previously!
+        line = f.readline(options[:row_sep])  # read one line
 
         # replace invalid byte sequence in UTF-8 with question mark to avoid errors
         line = line.force_encoding('utf-8').encode('utf-8', invalid: :replace, undef: :replace, replace: options[:invalid_byte_sequence]) if options[:force_utf8] || options[:file_encoding] !~ /utf-8/i
@@ -131,21 +127,22 @@ module SmarterCSV
         file_line_count += 1
         csv_line_count += 1
         print "processing file line %10d, csv line %10d\r" % [file_line_count, csv_line_count] if options[:verbose]
-        next  if  line =~ options[:comment_regexp]  # ignore all comment lines if there are any
+
+        next if options[:comment_regexp] && line =~ options[:comment_regexp] # ignore all comment lines if there are any
 
         # cater for the quoted csv data containing the row separator carriage return character
         # in which case the row data will be split across multiple lines (see the sample content in spec/fixtures/carriage_returns_rn.csv)
         # by detecting the existence of an uneven number of quote characters
-        multiline = line.count(options[:quote_char])%2 == 1
-        while line.count(options[:quote_char])%2 == 1
-          next_line = f.readline
+        multiline = line.count(options[:quote_char])%2 == 1 # should handle quote_char nil
+        while line.count(options[:quote_char])%2 == 1 # should handle quote_char nil
+          next_line = f.readline(options[:row_sep])
           next_line = next_line.force_encoding('utf-8').encode('utf-8', invalid: :replace, undef: :replace, replace: options[:invalid_byte_sequence]) if options[:force_utf8] || options[:file_encoding] !~ /utf-8/i
           line += next_line
           file_line_count += 1
         end
         print "\nline contains uneven number of quote chars so including content through file line %d\n" % file_line_count if options[:verbose] && multiline
 
-        line.chomp!    # will use $/ which is set to options[:col_sep]
+        line.chomp!(options[:row_sep])
 
         if (line =~ %r{#{options[:quote_char]}}) and (! options[:force_simple_split])
           dataA = begin
@@ -154,11 +151,17 @@ module SmarterCSV
             raise $!, "#{$!} [SmarterCSV: csv line #{csv_line_count}]", $!.backtrace
           end
         else
-          dataA =  line.split(options[:col_sep])
+          dataA = line.split(options[:col_sep], header_size)
         end
-####     dataA.map!{|x| x.gsub(%r/#{options[:quote_char]}/,'') }  # this is actually not a good idea as a default
-        dataA.map!{|x| x.strip}  if options[:strip_whitespace]
+        dataA.map!{|x| x.sub(/(#{options[:col_sep]})+\z/, '')} # remove any unwanted trailing col_sep characters at the end
+        dataA.map!{|x| x.strip} if options[:strip_whitespace]
+
+        # if all values are blank, then ignore this line
+        # SEE: https://github.com/rails/rails/blob/32015b6f369adc839c4f0955f2d9dce50c0b6123/activesupport/lib/active_support/core_ext/object/blank.rb#L121
+        next if options[:remove_empty_hashes] && blank?(dataA)
+
         hash = Hash.zip(headerA,dataA)  # from Facets of Ruby library
+
         # make sure we delete any key/value pairs from the hash, which the user wanted to delete:
         # Note: Ruby < 1.9 doesn't allow empty symbol literals!
         hash.delete(nil); hash.delete('');
@@ -166,18 +169,17 @@ module SmarterCSV
           eval('hash.delete(:"")')
         end
 
-        # remove empty values using the same regexp as used by the rails blank? method
-        # which caters for double \n and \r\n characters such as "1\r\n\r\n2" whereas the original check (v =~ /^\s*$/) does not
-        if options[:remove_empty_values]
+        if options[:remove_empty_values] == true
           if has_rails
             hash.delete_if{|k,v| v.blank?}
           else
-            hash.delete_if{|k,v| v.nil? || v !~ /[^[:space:]]/}
+            hash.delete_if{|k,v| blank?(v)}
           end
         end
 
         hash.delete_if{|k,v| ! v.nil? && v =~ /^(\d+|\d+\.\d+)$/ && v.to_f == 0} if options[:remove_zero_values]   # values are typically Strings!
         hash.delete_if{|k,v| v =~ options[:remove_values_matching]} if options[:remove_values_matching]
+
         if options[:convert_values_to_numeric]
           hash.each do |k,v|
             # deal with the :only / :except options to :convert_values_to_numeric
@@ -247,7 +249,6 @@ module SmarterCSV
         chunk = []  # initialize for next chunk of data
       end
     ensure
-      $/ = old_row_sep   # make sure this stupid global variable is always reset to it's previous value after we're done!
       f.close if f.respond_to?(:close)
     end
     if block_given?
@@ -258,8 +259,63 @@ module SmarterCSV
   end
 
   private
-  # acts as a road-block to limit processing when iterating over all k/v pairs of a CSV-hash:
 
+  def self.default_options
+    {
+      auto_row_sep_chars: 500,
+      chunk_size: nil ,
+      col_sep: ',',
+      comment_regexp: nil, # was: /\A#/,
+      convert_values_to_numeric: true,
+      downcase_header: true,
+      file_encoding: 'utf-8',
+      force_simple_split: false ,
+      force_utf8: false,
+      headers_in_file: true,
+      invalid_byte_sequence: '',
+      keep_original_headers: false,
+      key_mapping_hash: nil ,
+      quote_char: '"',
+      remove_empty_hashes: true ,
+      remove_empty_values: true,
+      remove_unmapped_keys: false,
+      remove_values_matching: nil,
+      remove_zero_values: false,
+      required_headers: nil,
+      row_sep: $INPUT_RECORD_SEPARATOR,
+      skip_lines: nil,
+      strings_as_keys: false,
+      strip_chars_from_headers: nil,
+      strip_whitespace: true,
+      user_provided_headers: nil,
+      value_converters: nil,
+      verbose: false,
+    }
+  end
+
+  def self.blank?(value)
+    case value
+    when Array
+      value.inject(true){|result, x| result &&= elem_blank?(x)}
+    when Hash
+      value.inject(true){|result, x| result &&= elem_blank?(x.last)}
+    else
+      elem_blank?(value)
+    end
+  end
+
+  def self.elem_blank?(value)
+    case value
+    when NilClass
+      true
+    when String
+      value !~ /\S/
+    else
+      false
+    end
+  end
+
+  # acts as a road-block to limit processing when iterating over all k/v pairs of a CSV-hash:
   def self.only_or_except_limit_execution( options, option_name, key )
     if options[option_name].is_a?(Hash)
       if options[option_name].has_key?( :except )
@@ -271,6 +327,24 @@ module SmarterCSV
     return false
   end
 
+  # raise exception if none is found
+  def self.guess_column_separator(filehandle, options)
+    del = [',', "\t", ';', ':', '|']
+    n = Hash.new(0)
+    5.times do
+      line = filehandle.readline(options[:row_sep])
+      del.each do |d|
+        n[d] += line.scan(d).count
+      end
+    rescue EOFError # short files
+      break
+    end
+    filehandle.rewind
+    raise SmarterCSV::NoColSepDetected if n.values.max == 0
+
+    col_sep = n.key(n.values.max)
+  end
+
   # limitation: this currently reads the whole file in before making a decision
   def self.guess_line_ending( filehandle, options )
     counts = {"\n" => 0 , "\r" => 0, "\r\n" => 0}
@@ -297,6 +371,8 @@ module SmarterCSV
       lines += 1
       break if options[:auto_row_sep_chars] && options[:auto_row_sep_chars] > 0 && lines >= options[:auto_row_sep_chars]
     end
+    filehandle.rewind
+
     counts["\r"] += 1 if last_char == "\r"
     # find the key/value pair with the largest counter:
     k,_ = counts.max_by{|_,v| v}

data/lib/smarter_csv/version.rb

@@ -1,3 +1,3 @@
 module SmarterCSV
-  VERSION = "1.3.0"
+  VERSION = "1.5.0"
 end

data/lib/smarter_csv.rb

@@ -1,3 +1,11 @@
+if ENV['COVERAGE']
+  require 'simplecov'
+  SimpleCov.start do
+    add_filter "/spec/"
+    add_filter "/pkg/"
+  end
+end
+
 require 'csv'
 require "smarter_csv/version"
 require "extensions/hash.rb"

data/smarter_csv.gemspec

@@ -1,21 +1,25 @@
 # -*- encoding: utf-8 -*-
 require File.expand_path('../lib/smarter_csv/version', __FILE__)
 
-Gem::Specification.new do |gem|
-  gem.authors       = ["Tilo Sloboda\n"]
-  gem.email         = ["tilo.sloboda@gmail.com\n"]
-  gem.description   = %q{Ruby Gem for smarter importing of CSV Files as Array(s) of Hashes, with optional features for processing large files in parallel, embedded comments, unusual field- and record-separators, flexible mapping of CSV-headers to Hash-keys}
-  gem.summary       = %q{Ruby Gem for smarter importing of CSV Files (and CSV-like files), with lots of optional features, e.g. chunked processing for huge CSV files}
-  gem.homepage      = "https://github.com/tilo/smarter_csv"
+Gem::Specification.new do |spec|
+  spec.name          = "smarter_csv"
+  spec.version       = SmarterCSV::VERSION
+  spec.authors       = ["Tilo Sloboda"]
+  spec.email         = ["tilo.sloboda@gmail.com"]
 
-  gem.files         = `git ls-files`.split($\)
-  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
-  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
-  gem.name          = "smarter_csv"
-  gem.require_paths = ["lib"]
-  gem.requirements  = ['csv'] # for CSV.parse() only needed in case we have quoted fields
-  gem.version       = SmarterCSV::VERSION
-  gem.licenses      = ['MIT','GPL-2'] # pick either of these licenses - only that license applies to you
-  gem.add_development_dependency "rspec"
-#  gem.add_development_dependency "guard-rspec"
+  spec.summary       = %q{Ruby Gem for smarter importing of CSV Files (and CSV-like files), with lots of optional features, e.g. chunked processing for huge CSV files}
+  spec.description   = %q{Ruby Gem for smarter importing of CSV Files as Array(s) of Hashes, with optional features for processing large files in parallel, embedded comments, unusual field- and record-separators, flexible mapping of CSV-headers to Hash-keys}
+  spec.homepage      = "https://github.com/tilo/smarter_csv"
+  spec.license       = 'MIT'
+
+  spec.files         = `git ls-files`.split($\)
+  spec.executables   = spec.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+  spec.requirements  = ['csv'] # for CSV.parse() only needed in case we have quoted fields
+  spec.add_development_dependency "rspec"
+  spec.add_development_dependency "simplecov"
+  #  spec.add_development_dependency "guard-rspec"
+
+  spec.metadata["homepage_uri"] = spec.homepage
 end

data/spec/fixtures/additional_separator.csv

@@ -0,0 +1,6 @@
+col1,col2
+eins,zwei
+uno,dos,
+one,two ,,,
+ichi, ,,,,,
+un

data/spec/fixtures/empty_columns_1.csv

@@ -0,0 +1,2 @@
+id,col1,col2,col3
+123,,,

data/spec/fixtures/empty_columns_2.csv

@@ -0,0 +1,2 @@
+id,col1,col2,col3
+123,,,1

data/spec/fixtures/hard_sample.csv

@@ -0,0 +1,2 @@
+Name,Email,Financial Status,Paid at,Fulfillment Status,Fulfilled at,Accepts Marketing,Currency,Subtotal,Shipping,Taxes,Total,Discount Code,Discount Amount,Shipping Method,Created at,Lineitem quantity,Lineitem name,Lineitem price,Lineitem compare at price,Lineitem sku,Lineitem requires shipping,Lineitem taxable,Lineitem fulfillment status,Billing Name,Billing Street,Billing Address1,Billing Address2,Billing Company,Billing City,Billing Zip,Billing Province,Billing Country,Billing Phone,Shipping Name,Shipping Street,Shipping Address1,Shipping Address2,Shipping Company,Shipping City,Shipping Zip,Shipping Province,Shipping Country,Shipping Phone,Notes,Note Attributes,Cancelled at,Payment Method,Payment Reference,Refunded Amount,Vendor,        rece,Tags,Risk Level,Source,Lineitem discount,Tax 1 Name,Tax 1 Value,Tax 2 Name,Tax 2 Value,Tax 3 Name,Tax 3 Value,Tax 4 Name,Tax 4 Value,Tax 5 Name,Tax 5 Value,Phone,Receipt Number,Duties,Billing Province Name,Shipping Province Name,Payment ID,Payment Terms Name,Next Payment Due At
+#MR1220817,foo@bar.com,paid,2022-02-08 22:31:28 +0100,unfulfilled,,yes,EUR,144,0,24,144,VIP,119.6,"Livraison Standard GRATUITE, 2-5 jours avec suivi",2022-02-08 22:31:26 +0100,2,Cire Épilation Nacrée,37,,WAX-200-NAC,true,true,pending,French Fry,64 Boulevard Budgié,64 Boulevard Budgié,,,dootdoot’,'49100,,FR,06 12 34 56 78,French Fry,64 Boulevard Budgi,64 Boulevard Budgié,,,dootdoot,'49100,,FR,06 12 34 56 78,,,,Stripe,c23800013619353.2,0,Goober  Rég,4331065802905,902,Low,web,0,FR TVA 20%,24,,,,,,,,,3366012111111,,,,,,,

data/spec/fixtures/numeric.csv

@@ -1,5 +1,5 @@
 First Name,Last Name,Reference, Wealth
 Dan,McAllister,0123,3.5
-,,,,,
+,,,
 Miles,O'Brian,2345,3
 Nancy,Homes,2345,01

data/spec/fixtures/separator_colon.csv

@@ -0,0 +1,4 @@
+Year:Make:Model:Length
+1997:Ford:E350:2,34
+2000:Mercury:Cougar:2,38
+2013:Tesla:Model S:4,97

data/spec/fixtures/separator_comma.csv

@@ -0,0 +1,4 @@
+Year,Make,Model,Length
+1997,Ford,E350,2.34
+2000,Mercury,Cougar,2.38
+2013,Tesla,Model S,4.97

data/spec/fixtures/separator_pipe.csv

@@ -0,0 +1,4 @@
+Year|Make|Model|Length
+1997|Ford|E350|2,34
+2000|Mercury|Cougar|2,38
+2013|Tesla|Model S|4,97

data/spec/fixtures/separator_tab.csv

@@ -0,0 +1,4 @@
+Year	Make	Model	Length
+1997	Ford	E350	2,34
+2000	Mercury	Cougar	2,38
+2013	Tesla	Model S	4,97

data/spec/smarter_csv/additional_separator_spec.rb

@@ -0,0 +1,45 @@
+require 'spec_helper'
+
+fixture_path = 'spec/fixtures'
+
+describe 'handling of additional trailing column separators' do
+  let(:file) { "#{fixture_path}/additional_separator.csv" }
+
+  describe '' do
+    let(:data) { SmarterCSV.process(file) }
+
+    it 'reads all lines' do
+      data.size.should eq 5
+    end
+
+    it 'reads regular lines' do
+      item = data[0]
+      item[:col1].should == 'eins'
+      item[:col2].should == 'zwei'
+    end
+
+    it 'strips single trailing col_sep character' do
+      item = data[1]
+      item[:col1].should == 'uno'
+      item[:col2].should == 'dos'
+    end
+
+    it 'strips multiple trailing col_sep characters' do
+      item = data[2]
+      item[:col1].should == 'one'
+      item[:col2].should == 'two'
+    end
+
+    it 'strips multiple trailing col_sep chars' do
+      item = data[3]
+      item[:col1].should == 'ichi'
+      item[:col2].should == nil
+    end
+
+    it 'strips multiple trailing col_sep chars' do
+      item = data[4]
+      item[:col1].should == 'un'
+      item[:col2].should == nil
+    end
+  end
+end

data/spec/smarter_csv/binary_file2_spec.rb

@@ -12,7 +12,7 @@ describe 'be_able_to' do
   it 'loads_binary_file_with_strings_as_keys' do 
     options = {:col_sep => "\cA", :row_sep => "\cB", :comment_regexp => /^#/, :strings_as_keys => true}
     data = SmarterCSV.process("#{fixture_path}/binary.csv", options)
-    data.flatten.size.should == 8
+    data.size.should == 8
     data.each do |item|
       # all keys should be strings
       item.keys.each{|x| x.class.should be == String}

data/spec/smarter_csv/blank_spec.rb

@@ -0,0 +1,55 @@
+require 'spec_helper'
+
+describe 'blank?' do
+  it 'is true for nil' do
+    SmarterCSV.send(:blank?, nil).should eq true
+  end
+
+  it 'is true for empty string' do
+    SmarterCSV.send(:blank?, '').should eq true
+  end
+
+  it 'is true for blank string' do
+    SmarterCSV.send(:blank?, '   ').should eq true
+  end
+
+  it 'is true for tab string' do
+    SmarterCSV.send(:blank?, " \t ").should eq true
+  end
+
+  it 'is false for string with content' do
+    SmarterCSV.send(:blank?, " 1 ").should eq false
+  end
+
+  it 'is false for numeic values' do
+    SmarterCSV.send(:blank?, 1).should eq false
+  end
+
+  describe 'arrays' do
+    it 'is true for empty arrays' do
+      SmarterCSV.send(:blank?, []).should eq true
+    end
+
+    it 'is true for blank arrays' do
+      SmarterCSV.send(:blank?, [nil, '', '  ', " \t "]).should eq true
+    end
+
+    it 'is false for non-blank arrays' do
+      SmarterCSV.send(:blank?, [nil, '', '  ', " 1 "]).should eq false
+    end
+  end
+
+  describe 'hashes' do
+    it 'is true for empty arrays' do
+      SmarterCSV.send(:blank?, {}).should eq true
+    end
+
+    it 'is true for blank arrays' do
+      SmarterCSV.send(:blank?, {a: nil, b: '', c: '  ', d: " \t "}).should eq true
+    end
+
+    it 'is false for non-blank arrays' do
+      SmarterCSV.send(:blank?, {a: nil, b: '', c: '  ', d: " 1 "}).should eq false
+    end
+  end
+end

data/spec/smarter_csv/carriage_return_spec.rb

@@ -3,7 +3,6 @@ require 'spec_helper'
 fixture_path = 'spec/fixtures'
 
 describe 'process files with line endings explicitly pre-specified' do
-
   it 'should process a file with \n for line endings and within data fields' do
     sep = "\n"
     options = {:row_sep => sep}
@@ -83,14 +82,14 @@ describe 'process files with line endings explicitly pre-specified' do
     data[1][:members].should == ["Jimmy Page", "Robert Plant", "John Bonham", "John Paul Jones"].join(text_sep)
     data[1][:albums].should == ["Led Zeppelin", "Led Zeppelin II", "Led Zeppelin III", "Led Zeppelin IV"].join(text_sep)
   end
-
 end
 
 describe 'process files with line endings in automatic mode' do
+  let(:options) { { row_sep: :auto } }
 
   it 'should process a file with \n for line endings and within data fields' do
     sep = "\n"
-    data = SmarterCSV.process("#{fixture_path}/carriage_returns_n.csv", {:row_sep => :auto})
+    data = SmarterCSV.process("#{fixture_path}/carriage_returns_n.csv", options)
     data.flatten.size.should == 8
     data[0][:name].should == "Anfield"
     data[0][:street].should == "Anfield Road"
@@ -112,7 +111,29 @@ describe 'process files with line endings in automatic mode' do
 
   it 'should process a file with \r for line endings and within data fields' do
     sep = "\r"
-    data = SmarterCSV.process("#{fixture_path}/carriage_returns_r.csv", {:row_sep => :auto})
+    data = SmarterCSV.process("#{fixture_path}/carriage_returns_r.csv", options)
+    data.flatten.size.should == 8
+    data[0][:name].should == "Anfield"
+    data[0][:street].should == "Anfield Road"
+    data[0][:city].should == "Liverpool"
+    data[1][:name].should == ["Highbury", "Highbury House"].join(sep)
+    data[2][:street].should == ["Sir Matt ", "Busby Way"].join(sep)
+    data[3][:city].should == ["Newcastle-upon-tyne ", "Tyne and Wear"].join(sep)
+    data[4][:name].should == ["White Hart Lane", "(The Lane)"].join(sep)
+    data[4][:street].should == ["Bill Nicholson Way ", "748 High Rd"].join(sep)
+    data[4][:city].should == ["Tottenham", "London"].join(sep)
+    data[5][:name].should == "Stamford Bridge"
+    data[5][:street].should == ["Fulham Road", "London"].join(sep)
+    data[5][:city].should be_nil
+    data[6][:name].should == ["Etihad Stadium", "Rowsley St", "Manchester"].join(sep)
+    data[7][:name].should == "Goodison"
+    data[7][:street].should == "Goodison Road"
+    data[7][:city].should == "Liverpool"
+  end
+
+  it 'also works when auto is given a string' do
+    sep = "\r"
+    data = SmarterCSV.process("#{fixture_path}/carriage_returns_r.csv", {row_sep: 'auto'})
     data.flatten.size.should == 8
     data[0][:name].should == "Anfield"
     data[0][:street].should == "Anfield Road"
@@ -134,7 +155,7 @@ describe 'process files with line endings in automatic mode' do
 
   it 'should process a file with \r\n for line endings and within data fields' do
     sep = "\r\n"
-    data = SmarterCSV.process("#{fixture_path}/carriage_returns_rn.csv", {:row_sep => :auto})
+    data = SmarterCSV.process("#{fixture_path}/carriage_returns_rn.csv", options)
     data.flatten.size.should == 8
     data[0][:name].should == "Anfield"
     data[0][:street].should == "Anfield Road"
@@ -157,7 +178,7 @@ describe 'process files with line endings in automatic mode' do
   it 'should process a file with more quoted text carriage return characters (\r) than line ending characters (\n)' do
     row_sep = "\n"
     text_sep = "\r"
-    data = SmarterCSV.process("#{fixture_path}/carriage_returns_quoted.csv", {:row_sep => :auto})
+    data = SmarterCSV.process("#{fixture_path}/carriage_returns_quoted.csv", options)
     data.flatten.size.should == 2
     data[0][:band].should == "New Order"
     data[0][:members].should == ["Bernard Sumner", "Peter Hook", "Stephen Morris", "Gillian Gilbert"].join(text_sep)
@@ -166,5 +187,4 @@ describe 'process files with line endings in automatic mode' do
     data[1][:members].should == ["Jimmy Page", "Robert Plant", "John Bonham", "John Paul Jones"].join(text_sep)
     data[1][:albums].should == ["Led Zeppelin", "Led Zeppelin II", "Led Zeppelin III", "Led Zeppelin IV"].join(text_sep)
   end
-
 end