smarter_csv 1.6.1 → 1.7.1

data/lib/smarter_csv/smarter_csv.rb

@@ -1,461 +0,0 @@
-module SmarterCSV
-  class SmarterCSVException < StandardError; end
-  class HeaderSizeMismatch < SmarterCSVException; end
-  class IncorrectOption < SmarterCSVException; end
-  class DuplicateHeaders < SmarterCSVException; end
-  class MissingHeaders < SmarterCSVException; end
-  class NoColSepDetected < SmarterCSVException; end
-  class KeyMappingError < SmarterCSVException; end
-  class MalformedCSVError < SmarterCSVException; end
-
-  # first parameter: filename or input object which responds to readline method
-  def SmarterCSV.process(input, options={}, &block)
-    options = default_options.merge(options)
-    options[:invalid_byte_sequence] = '' if options[:invalid_byte_sequence].nil?
-
-    headerA = []
-    result = []
-    @file_line_count = 0
-    @csv_line_count = 0
-    has_rails = !! defined?(Rails)
-    begin
-      fh = input.respond_to?(:readline) ? input : File.open(input, "r:#{options[:file_encoding]}")
-
-      # auto-detect the row separator
-      options[:row_sep] = SmarterCSV.guess_line_ending(fh, options) if options[:row_sep].to_sym == :auto
-      # attempt to auto-detect column separator
-      options[:col_sep] = guess_column_separator(fh, options) if options[:col_sep].to_sym == :auto
-
-      if (options[:force_utf8] || options[:file_encoding] =~ /utf-8/i) && ( fh.respond_to?(:external_encoding) && fh.external_encoding != Encoding.find('UTF-8') || fh.respond_to?(:encoding) && fh.encoding != Encoding.find('UTF-8') )
-        puts 'WARNING: you are trying to process UTF-8 input, but did not open the input with "b:utf-8" option. See README file "NOTES about File Encodings".'
-      end
-
-      if options[:skip_lines].to_i > 0
-        options[:skip_lines].to_i.times do
-          readline_with_counts(fh, options)
-        end
-      end
-
-      headerA, header_size = process_headers(fh, options)
-
-      # in case we use chunking.. we'll need to set it up..
-      if ! options[:chunk_size].nil? && options[:chunk_size].to_i > 0
-        use_chunks = true
-        chunk_size = options[:chunk_size].to_i
-        chunk_count = 0
-        chunk = []
-      else
-        use_chunks = false
-      end
-
-      # now on to processing all the rest of the lines in the CSV file:
-      while ! fh.eof?    # we can't use fh.readlines() here, because this would read the whole file into memory at once, and eof => true
-        line = readline_with_counts(fh, options)
-
-        # replace invalid byte sequence in UTF-8 with question mark to avoid errors
-        line = line.force_encoding('utf-8').encode('utf-8', invalid: :replace, undef: :replace, replace: options[:invalid_byte_sequence]) if options[:force_utf8] || options[:file_encoding] !~ /utf-8/i
-
-        print "processing file line %10d, csv line %10d\r" % [@file_line_count, @csv_line_count] if options[:verbose]
-
-        next if options[:comment_regexp] && line =~ options[:comment_regexp] # ignore all comment lines if there are any
-
-        # cater for the quoted csv data containing the row separator carriage return character
-        # in which case the row data will be split across multiple lines (see the sample content in spec/fixtures/carriage_returns_rn.csv)
-        # by detecting the existence of an uneven number of quote characters
-
-        multiline = line.count(options[:quote_char])%2 == 1 # should handle quote_char nil
-        while line.count(options[:quote_char])%2 == 1 # should handle quote_char nil
-          next_line = fh.readline(options[:row_sep])
-          next_line = next_line.force_encoding('utf-8').encode('utf-8', invalid: :replace, undef: :replace, replace: options[:invalid_byte_sequence]) if options[:force_utf8] || options[:file_encoding] !~ /utf-8/i
-          line += next_line
-          @file_line_count += 1
-        end
-        print "\nline contains uneven number of quote chars so including content through file line %d\n" % @file_line_count if options[:verbose] && multiline
-
-        line.chomp!(options[:row_sep])
-
-        dataA, data_size = parse(line, options, header_size)
-
-        dataA.map!{|x| x.strip} if options[:strip_whitespace]
-
-        # if all values are blank, then ignore this line
-        # SEE: https://github.com/rails/rails/blob/32015b6f369adc839c4f0955f2d9dce50c0b6123/activesupport/lib/active_support/core_ext/object/blank.rb#L121
-        next if options[:remove_empty_hashes] && blank?(dataA)
-
-        hash = Hash.zip(headerA,dataA)  # from Facets of Ruby library
-
-        # make sure we delete any key/value pairs from the hash, which the user wanted to delete:
-        # Note: Ruby < 1.9 doesn't allow empty symbol literals!
-        hash.delete(nil); hash.delete('');
-        if RUBY_VERSION.to_f > 1.8
-          eval('hash.delete(:"")')
-        end
-
-        if options[:remove_empty_values] == true
-          if has_rails
-            hash.delete_if{|k,v| v.blank?}
-          else
-            hash.delete_if{|k,v| blank?(v)}
-          end
-        end
-
-        hash.delete_if{|k,v| ! v.nil? && v =~ /^(\d+|\d+\.\d+)$/ && v.to_f == 0} if options[:remove_zero_values]   # values are typically Strings!
-        hash.delete_if{|k,v| v =~ options[:remove_values_matching]} if options[:remove_values_matching]
-
-        if options[:convert_values_to_numeric]
-          hash.each do |k,v|
-            # deal with the :only / :except options to :convert_values_to_numeric
-            next if SmarterCSV.only_or_except_limit_execution( options, :convert_values_to_numeric , k )
-
-            # convert if it's a numeric value:
-            case v
-            when /^[+-]?\d+\.\d+$/
-              hash[k] = v.to_f
-            when /^[+-]?\d+$/
-              hash[k] = v.to_i
-            end
-          end
-        end
-
-        if options[:value_converters]
-          hash.each do |k,v|
-            converter = options[:value_converters][k]
-            next unless converter
-            hash[k] = converter.convert(v)
-          end
-        end
-
-        next if hash.empty? if options[:remove_empty_hashes]
-
-        if use_chunks
-          chunk << hash  # append temp result to chunk
-
-          if chunk.size >= chunk_size || fh.eof?   # if chunk if full, or EOF reached
-            # do something with the chunk
-            if block_given?
-              yield chunk  # do something with the hashes in the chunk in the block
-            else
-              result << chunk  # not sure yet, why anybody would want to do this without a block
-            end
-            chunk_count += 1
-            chunk = []  # initialize for next chunk of data
-          else
-
-            # the last chunk may contain partial data, which also needs to be returned (BUG / ISSUE-18)
-
-          end
-
-          # while a chunk is being filled up we don't need to do anything else here
-
-        else # no chunk handling
-          if block_given?
-            yield [hash]  # do something with the hash in the block (better to use chunking here)
-          else
-            result << hash
-          end
-        end
-      end
-
-      # print new line to retain last processing line message
-      print "\n" if options[:verbose]
-
-      # last chunk:
-      if ! chunk.nil? && chunk.size > 0
-        # do something with the chunk
-        if block_given?
-          yield chunk  # do something with the hashes in the chunk in the block
-        else
-          result << chunk  # not sure yet, why anybody would want to do this without a block
-        end
-        chunk_count += 1
-        chunk = []  # initialize for next chunk of data
-      end
-    ensure
-      fh.close if fh.respond_to?(:close)
-    end
-    if block_given?
-      return chunk_count  # when we do processing through a block we only care how many chunks we processed
-    else
-      return result # returns either an Array of Hashes, or an Array of Arrays of Hashes (if in chunked mode)
-    end
-  end
-
-  private
-
-  def self.default_options
-    {
-      auto_row_sep_chars: 500,
-      chunk_size: nil ,
-      col_sep: ',',
-      comment_regexp: nil, # was: /\A#/,
-      convert_values_to_numeric: true,
-      downcase_header: true,
-      duplicate_header_suffix: nil,
-      file_encoding: 'utf-8',
-      force_simple_split: false ,
-      force_utf8: false,
-      headers_in_file: true,
-      invalid_byte_sequence: '',
-      keep_original_headers: false,
-      key_mapping_hash: nil ,
-      quote_char: '"',
-      remove_empty_hashes: true ,
-      remove_empty_values: true,
-      remove_unmapped_keys: false,
-      remove_values_matching: nil,
-      remove_zero_values: false,
-      required_headers: nil,
-      row_sep: $INPUT_RECORD_SEPARATOR,
-      skip_lines: nil,
-      strings_as_keys: false,
-      strip_chars_from_headers: nil,
-      strip_whitespace: true,
-      user_provided_headers: nil,
-      value_converters: nil,
-      verbose: false,
-    }
-  end
-
-  def self.readline_with_counts(filehandle, options)
-    line  = filehandle.readline(options[:row_sep])
-    @file_line_count += 1
-    @csv_line_count += 1
-    line
-  end
-
-  # parses a single line: either a CSV header and body line
-  # - quoting rules compared to RFC-4180 are somewhat relaxed
-  # - we are not assuming that quotes inside a fields need to be doubled
-  # - we are not assuming that all fields need to be quoted (0 is even)
-  # - works with multi-char col_sep
-  # - if header_size is given, only up to header_size fields are parsed
-  #
-  # We use header_size for parsing the body lines to make sure we always match the number of headers
-  # in case there are trailing col_sep characters in line
-  #
-  # Our convention is that empty fields are returned as empty strings, not as nil.
-  #
-  def self.parse(line, options, header_size = nil)
-    return [] if line.nil?
-
-    col_sep = options[:col_sep]
-    quote = options[:quote_char]
-    quote_count = 0
-    elements = []
-    start = 0
-    i = 0
-
-    while i < line.size do
-      if line[i...i+col_sep.size] == col_sep && quote_count.even?
-        break if !header_size.nil? && elements.size >= header_size
-
-        elements << cleanup_quotes(line[start...i], quote)
-        i += col_sep.size
-        start = i
-      else
-        quote_count += 1 if line[i] == quote
-        i += 1
-      end
-    end
-    elements << cleanup_quotes(line[start..-1], quote) if header_size.nil? || elements.size < header_size
-    [elements, elements.size]
-  end
-
-  def self.cleanup_quotes(field, quote)
-    return field if field.nil? || field !~ /#{quote}/
-
-    if field.start_with?(quote) && field.end_with?(quote)
-      field.delete_prefix!(quote)
-      field.delete_suffix!(quote)
-    end
-    field.gsub!("#{quote}#{quote}", quote)
-    field
-  end
-
-  def self.blank?(value)
-    case value
-    when Array
-      value.inject(true){|result, x| result &&= elem_blank?(x)}
-    when Hash
-      value.inject(true){|result, x| result &&= elem_blank?(x.last)}
-    else
-      elem_blank?(value)
-    end
-  end
-
-  def self.elem_blank?(value)
-    case value
-    when NilClass
-      true
-    when String
-      value !~ /\S/
-    else
-      false
-    end
-  end
-
-  # acts as a road-block to limit processing when iterating over all k/v pairs of a CSV-hash:
-  def self.only_or_except_limit_execution( options, option_name, key )
-    if options[option_name].is_a?(Hash)
-      if options[option_name].has_key?( :except )
-        return true if Array( options[ option_name ][:except] ).include?(key)
-      elsif options[ option_name ].has_key?(:only)
-        return true unless Array( options[ option_name ][:only] ).include?(key)
-      end
-    end
-    return false
-  end
-
-  # raise exception if none is found
-  def self.guess_column_separator(filehandle, options)
-    del = [',', "\t", ';', ':', '|']
-    n = Hash.new(0)
-    5.times do
-      line = filehandle.readline(options[:row_sep])
-      del.each do |d|
-        n[d] += line.scan(d).count
-      end
-    rescue EOFError # short files
-      break
-    end
-    filehandle.rewind
-    raise SmarterCSV::NoColSepDetected if n.values.max == 0
-
-    col_sep = n.key(n.values.max)
-  end
-
-  # limitation: this currently reads the whole file in before making a decision
-  def self.guess_line_ending( filehandle, options )
-    counts = {"\n" => 0 , "\r" => 0, "\r\n" => 0}
-    quoted_char = false
-
-    # count how many of the pre-defined line-endings we find
-    # ignoring those contained within quote characters
-    last_char = nil
-    lines = 0
-    filehandle.each_char do |c|
-      quoted_char = !quoted_char if c == options[:quote_char]
-      next if quoted_char
-
-      if last_char == "\r"
-        if c == "\n"
-          counts["\r\n"] +=  1
-        else
-          counts["\r"] += 1  # \r are counted after they appeared, we might
-        end
-      elsif c == "\n"
-        counts["\n"] += 1
-      end
-      last_char = c
-      lines += 1
-      break if options[:auto_row_sep_chars] && options[:auto_row_sep_chars] > 0 && lines >= options[:auto_row_sep_chars]
-    end
-    filehandle.rewind
-
-    counts["\r"] += 1 if last_char == "\r"
-    # find the key/value pair with the largest counter:
-    k,_ = counts.max_by{|_,v| v}
-    return k                    # the most frequent one is it
-  end
-
-  def self.raw_hearder
-    @raw_header
-  end
-
-  def self.headers
-    @headers
-  end
-
-  def self.process_headers(filehandle, options)
-    @raw_header = nil
-    @headers = nil
-    if options[:headers_in_file]        # extract the header line
-      # process the header line in the CSV file..
-      # the first line of a CSV file contains the header .. it might be commented out, so we need to read it anyhow
-      header = readline_with_counts(filehandle, options)
-      @raw_header = header
-
-      header = header.force_encoding('utf-8').encode('utf-8', invalid: :replace, undef: :replace, replace: options[:invalid_byte_sequence]) if options[:force_utf8] || options[:file_encoding] !~ /utf-8/i
-      header = header.sub(options[:comment_regexp],'') if options[:comment_regexp]
-      header = header.chomp(options[:row_sep])
-
-      header = header.gsub(options[:strip_chars_from_headers], '') if options[:strip_chars_from_headers]
-
-      file_headerA, file_header_size = parse(header, options)
-
-      file_headerA.map!{|x| x.gsub(%r/#{options[:quote_char]}/,'') }
-      file_headerA.map!{|x| x.strip}  if options[:strip_whitespace]
-      unless options[:keep_original_headers]
-        file_headerA.map!{|x| x.gsub(/\s+|-+/,'_')}
-        file_headerA.map!{|x| x.downcase }   if options[:downcase_header]
-      end
-    else
-      raise SmarterCSV::IncorrectOption , "ERROR: If :headers_in_file is set to false, you have to provide :user_provided_headers" unless options[:user_provided_headers]
-    end
-    if options[:user_provided_headers] && options[:user_provided_headers].class == Array && ! options[:user_provided_headers].empty?
-      # use user-provided headers
-      headerA = options[:user_provided_headers]
-      if defined?(file_header_size) && ! file_header_size.nil?
-        if headerA.size != file_header_size
-          raise SmarterCSV::HeaderSizeMismatch , "ERROR: :user_provided_headers defines #{headerA.size} headers !=  CSV-file #{input} has #{file_header_size} headers"
-        else
-          # we could print out the mapping of file_headerA to headerA here
-        end
-      end
-    else
-      headerA = file_headerA
-    end
-
-    # detect duplicate headers and disambiguate
-    headerA = process_duplicate_headers(headerA, options) if options[:duplicate_header_suffix]
-    header_size = headerA.size # used for splitting lines
-
-    headerA.map!{|x| x.to_sym } unless options[:strings_as_keys] || options[:keep_original_headers]
-
-    unless options[:user_provided_headers] # wouldn't make sense to re-map user provided headers
-      key_mappingH = options[:key_mapping]
-
-      # do some key mapping on the keys in the file header
-      #   if you want to completely delete a key, then map it to nil or to ''
-      if ! key_mappingH.nil? && key_mappingH.class == Hash && key_mappingH.keys.size > 0
-        # we can't map keys that are not there
-        missing_keys = key_mappingH.keys - headerA
-        puts "WARNING: missing header(s): #{missing_keys.join(",")}" unless missing_keys.empty?
-
-        headerA.map!{|x| key_mappingH.has_key?(x) ? (key_mappingH[x].nil? ? nil : key_mappingH[x]) : (options[:remove_unmapped_keys] ? nil : x)}
-      end
-    end
-
-    # header_validations
-    duplicate_headers = []
-    headerA.compact.each do |k|
-      duplicate_headers << k if headerA.select{|x| x == k}.size > 1
-    end
-    raise SmarterCSV::DuplicateHeaders , "ERROR: duplicate headers: #{duplicate_headers.join(',')}" unless duplicate_headers.empty?
-
-    if options[:required_headers] && options[:required_headers].is_a?(Array)
-      missing_headers = []
-      options[:required_headers].each do |k|
-        missing_headers << k unless headerA.include?(k)
-      end
-      raise SmarterCSV::MissingHeaders , "ERROR: missing headers: #{missing_headers.join(',')}" unless missing_headers.empty?
-    end
-
-    @headers = headerA
-    [headerA, header_size]
-  end
-
-  def self.process_duplicate_headers(headers, options)
-    counts = Hash.new(0)
-    result = []
-    headers.each do |key|
-      counts[key] += 1
-      if counts[key] == 1
-        result << key
-      else
-        result << [key, options[:duplicate_header_suffix], counts[key]].join
-      end
-    end
-    result
-  end
-end

data/spec/fixtures/additional_separator.csv

@@ -1,6 +0,0 @@
-col1,col2
-eins,zwei
-uno,dos,
-one,two ,,,
-ichi, ,,,,,
-un

data/spec/fixtures/basic.csv

@@ -1,8 +0,0 @@
-First Name,Last Name,Dogs,Cats,Birds,Fish
-Dan,McAllister,2,0,,
-Lucy,Laweless,,5,0,
-,,,,,
-Miles,O'Brian,0,0,0,21
-Nancy,Homes,2,0,1, 
-Hernán,Curaçon,3,0,0,
-,,,,,

data/spec/fixtures/binary.csv

@@ -1 +0,0 @@
-#timestamp
item_id
parent_id
name#some comments here#these can be multiple comments in the header section#even more comments here1381388409
10
1
Thing 11381388409
11
1
Thing 21381388409
12
1
Thing 31381388409
1

Parent 11381388409
2

Parent 21381388409
20
2
Thing 41381388409
21
2
Thing 51381388409
22
2
Thing 6

data/spec/fixtures/carriage_returns_n.csv

@@ -1,18 +0,0 @@
-Name,Street,City
-Anfield,Anfield Road,Liverpool
-"Highbury
-Highbury House",75 Drayton Park,London
-Old Trafford,"Sir Matt 
-Busby Way",Manchester
-St. James' Park,,"Newcastle-upon-tyne 
-Tyne and Wear"
-"White Hart Lane
-(The Lane)","Bill Nicholson Way 
-748 High Rd","Tottenham
-London"
-Stamford Bridge,"Fulham Road
-London",
-"Etihad Stadium
-Rowsley St
-Manchester",,
-Goodison,Goodison Road,Liverpool

data/spec/fixtures/carriage_returns_quoted.csv

@@ -1,3 +0,0 @@
-Band,Members,Albums
-New Order,"Bernard Sumner
-Led Zeppelin,"Jimmy Page

data/spec/fixtures/carriage_returns_r.csv

@@ -1 +0,0 @@
-Name,Street,City

data/spec/fixtures/carriage_returns_rn.csv

@@ -1,18 +0,0 @@
-Name,Street,City
-Anfield,Anfield Road,Liverpool
-"Highbury
-Highbury House",75 Drayton Park,London
-Old Trafford,"Sir Matt 
-Busby Way",Manchester
-St. James' Park,,"Newcastle-upon-tyne 
-Tyne and Wear"
-"White Hart Lane
-(The Lane)","Bill Nicholson Way 
-748 High Rd","Tottenham
-London"
-Stamford Bridge,"Fulham Road
-London",
-"Etihad Stadium
-Rowsley St
-Manchester",,
-Goodison,Goodison Road,Liverpool

data/spec/fixtures/chunk_cornercase.csv

@@ -1,10 +0,0 @@
-a,b,c
-1,2,3
-,,
-4,5,6
-7,8,9
-,,
-10,11,12
-,,
-13,14,15
-,,

data/spec/fixtures/duplicate_headers.csv

@@ -1,3 +0,0 @@
-email,firstname,lastname,email,age
-tom@bla.com,Tom,Sawyer,mike@bla.com,34
-eri@bla.com,Eri,Chan,tom@bla.com,21

data/spec/fixtures/empty.csv

@@ -1,5 +0,0 @@
-not empty 1,not empty 2,not empty 3,empty 1,empty 2
-56
-
-666
-",?,"

data/spec/fixtures/empty_columns_1.csv

@@ -1,2 +0,0 @@
-id,col1,col2,col3
-123,,,

data/spec/fixtures/empty_columns_2.csv

@@ -1,2 +0,0 @@
-id,col1,col2,col3
-123,,,1

data/spec/fixtures/hard_sample.csv

@@ -1,2 +0,0 @@
-Name,Email,Financial Status,Paid at,Fulfillment Status,Fulfilled at,Accepts Marketing,Currency,Subtotal,Shipping,Taxes,Total,Discount Code,Discount Amount,Shipping Method,Created at,Lineitem quantity,Lineitem name,Lineitem price,Lineitem compare at price,Lineitem sku,Lineitem requires shipping,Lineitem taxable,Lineitem fulfillment status,Billing Name,Billing Street,Billing Address1,Billing Address2,Billing Company,Billing City,Billing Zip,Billing Province,Billing Country,Billing Phone,Shipping Name,Shipping Street,Shipping Address1,Shipping Address2,Shipping Company,Shipping City,Shipping Zip,Shipping Province,Shipping Country,Shipping Phone,Notes,Note Attributes,Cancelled at,Payment Method,Payment Reference,Refunded Amount,Vendor,        rece,Tags,Risk Level,Source,Lineitem discount,Tax 1 Name,Tax 1 Value,Tax 2 Name,Tax 2 Value,Tax 3 Name,Tax 3 Value,Tax 4 Name,Tax 4 Value,Tax 5 Name,Tax 5 Value,Phone,Receipt Number,Duties,Billing Province Name,Shipping Province Name,Payment ID,Payment Terms Name,Next Payment Due At
-#MR1220817,foo@bar.com,paid,2022-02-08 22:31:28 +0100,unfulfilled,,yes,EUR,144,0,24,144,VIP,119.6,"Livraison Standard GRATUITE, 2-5 jours avec suivi",2022-02-08 22:31:26 +0100,2,Cire Épilation Nacrée,37,,WAX-200-NAC,true,true,pending,French Fry,64 Boulevard Budgié,64 Boulevard Budgié,,,dootdoot’,'49100,,FR,06 12 34 56 78,French Fry,64 Boulevard Budgi,64 Boulevard Budgié,,,dootdoot,'49100,,FR,06 12 34 56 78,,,,Stripe,c23800013619353.2,0,Goober  Rég,4331065802905,902,Low,web,0,FR TVA 20%,24,,,,,,,,,3366012111111,,,,,,,

data/spec/fixtures/ignore_comments.csv

@@ -1,11 +0,0 @@
-not a comment#First Name,Last Name,Dogs,Cats,Birds,Fish
-# comment two
-Dan,McAllister,2,0,,
-Lucy#L,Laweless,,5,0,
-# anothter comment
-,,,,,
-Miles,O'Brian,0,0,0,21
-Nancy,Homes,2,0,1,
-Hernán,Curaçon,3,0,0,
-#comment,comment,1,2,3,4
-,,,,,

data/spec/fixtures/ignore_comments2.csv

@@ -1,3 +0,0 @@
-h1,h2
-a,"b
-#c"

data/spec/fixtures/key_mapping.csv

@@ -1,2 +0,0 @@
-THIS,THAT,other
-this,that,other

data/spec/fixtures/line_endings_n.csv

@@ -1,4 +0,0 @@
-name,count,price
-hammer,4,12.50
-axe,2,7.30
-crowbar,3,17.50

data/spec/fixtures/line_endings_r.csv

@@ -1 +0,0 @@
-name,count,price

data/spec/fixtures/line_endings_rn.csv

@@ -1,4 +0,0 @@
-name,count,price
-hammer,4,12.50
-axe,2,7.30
-crowbar,3,17.50

data/spec/fixtures/lots_of_columns.csv

@@ -1,2 +0,0 @@
-Column 0,Column 1,Column 2,Column 3,Column 4,Column 5,Column 6,Column 7,Column 8,Column 9,Column 10,Column 11,Column 12,Column 13,Column 14,Column 15,Column 16,Column 17,Column 18,Column 19,Column 20,Column 21,Column 22,Column 23,Column 24,Column 25,Column 26,Column 27,Column 28,Column 29,Column 30,Column 31,Column 32,Column 33,Column 34,Column 35,Column 36,Column 37,Column 38,Column 39,Column 40,Column 41,Column 42,Column 43,Column 44,Column 45,Column 46,Column 47,Column 48,Column 49,Column 50,Column 51,Column 52,Column 53,Column 54,Column 55,Column 56,Column 57,Column 58,Column 59,Column 60,Column 61,Column 62,Column 63,Column 64,Column 65,Column 66,Column 67,Column 68,Column 69,Column 70,Column 71,Column 72,Column 73,Column 74,Column 75,Column 76,Column 77,Column 78,Column 79,Column 80,Column 81,Column 82,Column 83,Column 84,Column 85,Column 86,Column 87,Column 88,Column 89,Column 90,Column 91,Column 92,Column 93,Column 94,Column 95,Column 96,Column 97,Column 98,Column 99,Column 100,Column 101,Column 102,Column 103,Column 104,Column 105,Column 106,Column 107,Column 108,Column 109,Column 110,Column 111,Column 112,Column 113,Column 114,Column 115,Column 116,Column 117,Column 118,Column 119,Column 120,Column 121,Column 122,Column 123,Column 124,Column 125,Column 126,Column 127,Column 128,Column 129,Column 130,Column 131,Column 132,Column 133,Column 134,Column 135,Column 136,Column 137,Column 138,Column 139,Column 140,Column 141,Column 142,Column 143,Column 144,Column 145,Column 146,Column 147,Column 148,Column 149,Column 150,Column 151,Column 152,Column 153,Column 154,Column 155,Column 156,Column 157,Column 158,Column 159,Column 160,Column 161,Column 162,Column 163,Column 164,Column 165,Column 166,Column 167,Column 168,Column 169,Column 170,Column 171,Column 172,Column 173,Column 174,Column 175,Column 176,Column 177,Column 178,Column 179,Column 180,Column 181,Column 182,Column 183,Column 184,Column 185,Column 186,Column 187,Column 188,Column 189,Column 190,Column 191,Column 192,Column 193,Column 194,Column 195,Column 196,Column 197,Column 198,Column 199,Column 200,Column 201,Column 202,Column 203,Column 204,Column 205,Column 206,Column 207,Column 208,Column 209,Column 210,Column 211,Column 212,Column 213,Column 214,Column 215,Column 216,Column 217,Column 218,Column 219,Column 220,Column 221,Column 222,Column 223,Column 224,Column 225,Column 226,Column 227,Column 228,Column 229,Column 230,Column 231,Column 232,Column 233,Column 234,Column 235,Column 236,Column 237,Column 238,Column 239,Column 240,Column 241,Column 242,Column 243,Column 244,Column 245,Column 246,Column 247,Column 248,Column 249,Column 250,Column 251,Column 252,Column 253,Column 254,Column 255,Column 256,Column 257,Column 258,Column 259,Column 260,Column 261,Column 262,Column 263,Column 264,Column 265,Column 266,Column 267,Column 268,Column 269,Column 270,Column 271,Column 272,Column 273,Column 274,Column 275,Column 276,Column 277,Column 278,Column 279,Column 280,Column 281,Column 282,Column 283,Column 284,Column 285,Column 286,Column 287,Column 288,Column 289,Column 290,Column 291,Column 292,Column 293,Column 294,Column 295,Column 296,Column 297,Column 298,Column 299,Column 300,Column 301,Column 302,Column 303,Column 304,Column 305,Column 306,Column 307,Column 308,Column 309,Column 310,Column 311,Column 312,Column 313,Column 314,Column 315,Column 316,Column 317,Column 318,Column 319,Column 320,Column 321,Column 322,Column 323,Column 324,Column 325,Column 326,Column 327,Column 328,Column 329,Column 330,Column 331,Column 332,Column 333,Column 334,Column 335,Column 336,Column 337,Column 338,Column 339,Column 340,Column 341,Column 342,Column 343,Column 344,Column 345,Column 346,Column 347,Column 348,Column 349,Column 350,Column 351,Column 352,Column 353,Column 354,Column 355,Column 356,Column 357,Column 358,Column 359,Column 360,Column 361,Column 362,Column 363,Column 364,Column 365,Column 366,Column 367,Column 368,Column 369,Column 370,Column 371,Column 372,Column 373,Column 374,Column 375,Column 376,Column 377,Column 378,Column 379,Column 380,Column 381,Column 382,Column 383,Column 384,Column 385,Column 386,Column 387,Column 388,Column 389,Column 390,Column 391,Column 392,Column 393,Column 394,Column 395,Column 396,Column 397,Column 398,Column 399,Column 400,Column 401,Column 402,Column 403,Column 404,Column 405,Column 406,Column 407,Column 408,Column 409,Column 410,Column 411,Column 412,Column 413,Column 414,Column 415,Column 416,Column 417,Column 418,Column 419,Column 420,Column 421,Column 422,Column 423,Column 424,Column 425,Column 426,Column 427,Column 428,Column 429,Column 430,Column 431,Column 432,Column 433,Column 434,Column 435,Column 436,Column 437,Column 438,Column 439,Column 440,Column 441,Column 442,Column 443,Column 444,Column 445,Column 446,Column 447,Column 448,Column 449,Column 450,Column 451,Column 452,Column 453,Column 454,Column 455,Column 456,Column 457,Column 458,Column 459,Column 460,Column 461,Column 462,Column 463,Column 464,Column 465,Column 466,Column 467,Column 468,Column 469,Column 470,Column 471,Column 472,Column 473,Column 474,Column 475,Column 476,Column 477,Column 478,Column 479,Column 480,Column 481,Column 482,Column 483,Column 484,Column 485,Column 486,Column 487,Column 488,Column 489,Column 490,Column 491,Column 492,Column 493,Column 494,Column 495,Column 496,Column 497,Column 498,Column 499
-rFDAsZLKkINpb,ChLVlxWpwZfrOZg,uGJbxzrFncG,R,P,j,QoKRekBnucMA,N,KJFTwrG,Fs,pKazVnOVFnLvAcK,bo,EgJwvAwnGYPdvj,qVPGxupPKQI,FtNJBxxMIBxezfS,,qrcOm,Angkc,kwhJvozx,H,Yk,Bs,JZTVzstyk,sDiZuoBOVPA,hsgzXcqSnUaF,,AMQcprcuLABRvLF,NxwQvVApPdtTJSw,tKjftA,lOyqAtUuXQtK,ImgNiMBPRo,dyVW,dVbtwSuZuX,IQN,pUEEshP,QVKcImQ,HDoRZ,jrQpVdvtjGHv,YkP,dHyjESzVsgUdRh,p,qz,oCP,clMCAiuiw,JzxVq,zpn,bwfoYmrJkTc,t,HGYcQdMmNjb,CFFbJTeOozA,,MwroljZZwEoBOoc,,wkIRXssJZ,LSaIDarn,qky,BDMJVYE,thOpnBTTiGMfa,JVhoOwTdoukH,xIECafnWcE,,,kbxzc,viKcs,dwVMljnI,fYXJYw,PhohmrDa,alGbl,aeqwrXgjr,MJ,gapeNlzKYUP,tys,PJPfxFYYFPclGJV,HIUXf,AjmDljNZjYqB,HbxoDVzzkQ,aLQFk,PormYxT,rATMrIlCFrDwOzD,Krv,pMwcHztVfVkKo,srb,fdlXgWtPK,r,UZvVNkLYigVzZO,RPDHskEaMqq,dVkUXNKFRbwq,FGSqF,mqG,aZzufH,LHsN,NkKlcjoRFTbFH,ChEHvIQDYomC,L,URFkXirNgU,M,gAtCyH,FLkbLzwEBLgtlt,OadGQgGxGqGkT,tEQUWJDpb,p,fXIyPxIftbgHNPL,dM,mrz,cGXxRFLUG,MMvNyk,mKEcw,QMROkjgOWnBSZI,DQQ,QeKZt,Xaor,KUPKflOhGnrOMu,VpwHZUN,raDbObrwsRaa,,WqddHDTh,wj,XZtQIS,CGPUgYOQtiu,,zRkl,,ATlS,JheeokutCEtvAdV,hozEKLGRET,DsEiWDjQ,lTQjDhEah,WqREQoJLBSP,,FOaImBDZclTMM,vhEjpJddIVh,xC,ZCSpDtdJw,phRxoGO,VdhVyRebsNynrk,HfCkVLQnLnRLA,dwxQUCPZc,koFg,lumNQmIweYyTc,R,bDRcfIO,cxH,n,navCHjPdfj,puDQpjuIIMRDqK,EVkAd,ARVpVku,hUEqyejzKt,,Qtygnqk,HlZgSRnbmIHZP,QZpVTn,OQgoIZhxIsh,TYPUMV,ofjg,YvYMWpSOAA,PGtCCHInmFBqPp,va,viWLIIjBOEwQCP,E,yjKVemUmGC,FQhSaVdeAJjO,flcBAZzLOlrg,bShmRGHdOemzMZi,gjUzxDONxWymoK,ZS,VBFVHug,igVYBKUJIcGKbim,msrUQUHTNymL,,pVeEHwNN,BqnU,UkYsIhEnqayMIa,RQHqYCoEU,ujuLHy,DctKlTNlADp,MJmfaBxeNIONUTQ,BcPqmfBSWJEs,kH,bYQiLOweBh,ajEmwTPgdGtDI,aeMJiQthn,QfvLiAuJjYB,aNauvl,bcCqdQie,tCwTqF,tNVw,sbjLwbktZUM,Mkrlh,sQihM,clRevuPt,weuxLMDulNP,juaeGRXAz,SKenpQsEZqAA,UGdQdLHvOH,brBHqPjs,WLS,fZ,DBSBUMvgLYzbKH,eFks,zLt,hqnDnIsUsremb,dizzYLAKUGXsH,vDWNIbiepsMPPm,QHNpZgZ,vl,KpukrVVjLDll,rbOPLzI,woALqQUQnt,JiG,SGZGCjPyvceLmy,ofcCQgXjgVmRsxV,,EERhEDO,PxuJxrOka,ohMLIz,vpUnwsJejGkurJ,ISjplKfjrYLE,jZgXFIcRDQQkcI,XfKGtVsclDH,UtKeOSPhbf,,AtWJyGatftYCiA,pzEJCjFA,beuwjh,OrrcIHW,SjuBT,xVNUQEjxyB,CTDOUCAxhx,YpCApsrXvk,Gp,Us,ALBNnNFhdbMYjpl,msB,XniwYCr,aMImLpFlzbz,p,guQBgEARv,NRwcXFoVyjuRi,QfLRUsEQq,WrzdF,QVb,oDTGOMYEVu,iAfcPl,wLkI,VON,,P,TPwCynToFkJI,gZzcqx,QrZNqo,p,WSfbj,ldxi,taws,meZ,g,icjnchZcQQIqrnQ,RQRmriLCPE,Lxnxu,AoC,XNbQ,GhQqR,GSHHsytQGKNLJGU,PBi,IjIszY,WfXyRMqGBkI,jL,loeFjtFErHKhPv,YHAubVlaujA,hNAbVNu,sLhChzeLLvYXR,DnL,hamyOyGAouk,cNOZnSPNhoFjAFq,RXOFzh,JkgdgUCEI,ZidLIxtn,KaMuBFTdvSSN,Owj,dSHiNUxjaGQcieG,rVSK,DwxWsgHgbeiIVoF,I,NsOEAk,Hqdv,Xeedwold,aVFPYNWIehmpgZ,gH,yl,GDesAPfYPtrkDEx,YZbPYzQBUaseUBi,FYSJAlciOmnK,DjRdbezjrAdQcX,FUNKjgklfnksvn,NMCfDtqFvcI,T,rpkwYAmR,nTAJTyiVPbFuv,qKdFCrPCEPRE,xATVXeqsAKRcYV,tHzCGeqnplni,cXGiZMgXct,souXHmYybMmbyhg,i,oDhWt,L,kwesSP,jyksdMdyne,TCyYt,wX,JiSsPdDajKirPgL,cc,qnCQz,TS,luVvIgALvyB,,eGpRJq,RKWwtfwHo,ISEH,SiRldWBiF,jyFuxIz,cVojag,FGWd,HF,xI,xOwUyr,iDDEDH,vyDxjfVus,zLvpx,ssrFvXDVo,kXEzRcW,opsKOmCLbFwSM,xHfeHRWwalx,n,KOrSY,CAOTgyQJm,wCuuuzwffVEY,ldmjKUum,EDQzj,czqFC,GYAKCxOiu,nAM,uNQnVaBCj,TvoVlpJWYI,SFlUDJ,XTuQjsZtpUhgWId,D,xEiowDk,PaBftY,,PNdZMqagJawiLKy,pUcrcOEKSP,RlDCo,,kcK,RJugqQIBYfhC,W,ov,PzfdKIR,OBIa,JYXxqRucy,UaTDgAkJfw,,ydlnbzoww,fQdyIegvIc,DlT,LSqWLh,ihm,sR,MInOCbVexl,NuwJxdxvkTXa,nlJrPpKDIf,iWOwycGEID,PQdoTHebXj,DROZTCRnma,cFAMtx,BkYpK,JitcTdIPlTMEfl,wOSBOJEnIOw,CeEItVPQEeIfFgZ,OiCeEuVRKJ,hjFaJxsTYRCgFYz,giSKHpCNXNk,RaNmFDao,htRSneNtuDta,HdRaC,vkIkWSGebQbpaT,kzyJVn,gMMGxfRjFskmoo,xtfUXOdrlUs,CoSIbIPfVfTXko,zVOXIJ,UGQIlWuRzDJAPsn,,sWXjt,dNkgPEgim,WVmBPZSTa,KJnedQffhg,e,XhgwuoHiR,rt,FYNI,Rk,ImHT,QXR,Ta,wvU,qhsGIux,yyrLPMQmKVkCp,SBYM,a,aCxwZKlj,,lcNXLvIEkjQTO,XjpXtlM,PhdrxqxDVWO,pCIQtTuwu,WAbZHWAEtzLJOql,XDETKm,KuF,PozCBInp,d,tWtj,QKjx,eFevbQsOtGGxJM,FZIvXPnOKDMDgdB,khe,YdnPTMxczkm,L,SREiJirL,irZJxq,vl,SYsHw,vKSe,ypvRthKFyTRs,IuAdRz,lGotaaKUU,hxMTxzrUFT,dFCbGN,WDTco,xXhedTTXbbDllWX,M,MrhdNyXzFxP,LjX,qRqvrGrNzC,E,nQ,gOWQSgeCwVIq,ASGxcPYYmTAVm,gbnOqDOicyz,h,gviqOP,oV,QjDOngDXFeYp,ojPwZPSodxzxbu,jcWTp,ERHZmFboGYVCQxP,ox,PsFrmpSe,GbhohxlLVcU,rKMLpQNsvH,UzMJMaK,GilkAEVQWzUA,PGCrmxvKB,jOEDmSM,arTAqyNOUtsWj,,,sQXstIGqFfJg,FYOsFdUYr,DwYhqnBlsGtfez,FcKkwqG,sqZFwfyPByL,RkB,tYVdRtbHpdTRPBf,kObsAtqgghqe,,awzrSqHxz,,aXarXMPnQJgqJyM,ldb,NOdwJ,gUYvxxVPCcrmDOd,dcNR,,uglxXybrDyaHCU,JYRLQC,SoolzzZaZ,OZXZiMSimJrH,iZQiZl,bIDq,OSd,UopfHT,f,v,ynUYQwMmKrH,MtF,Cn,ad

data/spec/fixtures/malformed.csv

@@ -1,3 +0,0 @@
-"name","dob"
-"Arnold Schwarzenegger","1947-07-30"
-"Jeff "the dude" Bridges","1949-12-04"

data/spec/fixtures/malformed_header.csv

@@ -1,3 +0,0 @@
-"name","dob"dob""
-"Arnold Schwarzenegger","1947-07-30"
-"Jeff Bridges","1949-12-04"

data/spec/fixtures/money.csv

@@ -1,3 +0,0 @@
-item,price
-Book,$9.99
-Mug,$14.99

data/spec/fixtures/no_header.csv

@@ -1,7 +0,0 @@
-Dan,McAllister,2,0,,
-Lucy,Laweless,,5,0,
-,,,,,
-Miles,O'Brian,0,0,0,21
-Nancy,Homes,2,0,1, 
-Hernán,Curaçon,3,0,0,
-,,,,,

data/spec/fixtures/numeric.csv

@@ -1,5 +0,0 @@
-First Name,Last Name,Reference, Wealth
-Dan,McAllister,0123,3.5
-,,,
-Miles,O'Brian,2345,3
-Nancy,Homes,2345,01

data/spec/fixtures/pets.csv

@@ -1,5 +0,0 @@
-first name,last name,dogs,cats,birds,fish
-Dan,McAllister,2,,,
-Lucy,Laweless,,5,,
-Miles,O'Brian,,,,21
-Nancy,Homes,2,,1,