wtf_csv 1.0.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 94b4fdd4e3201ed522d4867b95a5d93d4327b00e
+  data.tar.gz: 6782a9fe9282463989816d7532dbd6f5862412bd
+SHA512:
+  metadata.gz: d28528baac8f1d4dec98c5085a363595ad6930710d28d46f64dda87398b3ec872149c0bab36a8f0fe761953d751e00097632b5a62e23b5e7553127c26715b8a8
+  data.tar.gz: 77ab306c01fc249efe80b2f27976475da2c111348d490257ec2a2188ae5404b2f2bcb320ff3f36a0f5b6e071523055becaac752212015d055e460ecb3a4d825b

data/.gitignore

@@ -0,0 +1,4 @@
+*.json
+lib/*.csv
+*.py
+*.gem

data/.rspec

@@ -0,0 +1,2 @@
+--color
+--require spec_helper

data/Gemfile

@@ -0,0 +1,3 @@
+source 'https://rubygems.org'
+  # required gems in .gemspec
+gemspec

data/Gemfile.lock

@@ -0,0 +1,31 @@
+PATH
+  remote: .
+  specs:
+    wtf_csv (0.0.0)
+      smarter_csv
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    diff-lcs (1.2.5)
+    rspec (3.3.0)
+      rspec-core (~> 3.3.0)
+      rspec-expectations (~> 3.3.0)
+      rspec-mocks (~> 3.3.0)
+    rspec-core (3.3.2)
+      rspec-support (~> 3.3.0)
+    rspec-expectations (3.3.1)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.3.0)
+    rspec-mocks (3.3.2)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.3.0)
+    rspec-support (3.3.0)
+    smarter_csv (1.1.0)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  rspec
+  wtf_csv!

data/README.md

@@ -0,0 +1,44 @@
+# wtf_csv
+Ruby gem to detect formatting issues in a CSV
+
+The CSV file format is meant to be an easy way to transport data. Anyone who has had to maintain an import process, however, knows that it's easy to mess up. Usually the entire landscape looks like this:
+  1. An importer expects CSV files to be provided in some specific format
+  2. The files are given in a different format
+  3. The import fails; or even worse, the import succeeds but the data is mangled
+  4. Some poor souls must dig through the CSV file to figure out what happened. Usually issues are related to bad cell quoting, inconsistent column counts, etc.
+  
+This gem seeks to make this process less terrible by providing a way to easily surface common formatting issues on a CSV file.
+
+`WtfCSV.scan` has the following options:
+```
+|---------------------------------|----------|--------------------------------------------------------------------------------------|
+| Option                          | Default  |  Explanation                                                                         |
+|---------------------------------|----------|--------------------------------------------------------------------------------------|
+| :col_sep                        |   ','    | Column separator                                                                     |
+| :row_sep                        | $/ ,"\n" | Row separator - defaults to system's $/ , which defaults to "\n"                     |
+|                                 |          | This can also be set to :auto, but will process the whole cvs file first  (slow!)    |
+| :quote_char                     |   '"'    | Quotation character                                                                  |
+| :escape_char                    |   '\'    | Character to escape quotes                                                           |
+|---------------------------------|----------|--------------------------------------------------------------------------------------|
+| :check_col_count                |   true   | If set, checks for issues in the number of columns that are present                  |
+| :num_cols                       |    0     | If :check_col_count is set and this value is non-zero, will return errors if any     |
+|                                 |          | line does not have this number of columns                                            |
+| :col_threshold                  |    80    | If :check_col_count is set, this is the percentage of rows that must have a column   |
+|                                 |          | count in order for the module to assume this is the target number of columns.        |
+|                                 |          |   For example, if there are 10 line in the file, and this value is set to 80, then   |
+|                                 |          |   at least 8 lines must have a certain number of columns for the module to assume    |
+|                                 |          |   this is the number of columns that rows are supposed to have                       |
+|---------------------------------|----------|--------------------------------------------------------------------------------------|
+| :ignore_string                  |   nil    | If a line is equal to this string, the line will not be checked for issues           |
+|---------------------------------|----------|--------------------------------------------------------------------------------------|
+| :allow_row_sep_in_quoted_fields |  false   | Allows :row_sep characters to be present in quoted fields. Otherwise if there are    |
+|                                 |          | line ending characters in a field, they will be treat as sequential lines and you'll |
+|                                 |          | likely receive column count errors (if you're checking for them)                     |
+|---------------------------------|----------|--------------------------------------------------------------------------------------|
+| :max_chars_in_field             |   nil    | Ensures that fields have less than or equal to the provided number of characters     |
+|---------------------------------|----------|--------------------------------------------------------------------------------------|
+| :file_encoding                  | 'utf-8'  | Set the file encoding                                                                |
+|---------------------------------|----------|--------------------------------------------------------------------------------------|
+```
+
+If you happen upon this, know that this is in development - though should be very stable at this point. Soon this will be available on https://rubygems.org, where you'll be able to install with `gem install wtf_csv` or putting `require 'wtf_csv'` in your gemfile. Until then, feel free to install from source and bundle it into a gem yourself - just give credit where it is due.

data/Rakefile

@@ -0,0 +1,19 @@
+#!/usr/bin/env rake
+require "bundler/gem_tasks"
+
+require 'rubygems'
+require 'rake'
+
+require 'rspec/core/rake_task'
+
+desc "Run RSpec"
+RSpec::Core::RakeTask.new do |t|
+  t.verbose = false
+end
+
+desc "Run specs for all test cases"
+task :spec_all do 
+  system "rake spec"
+end
+
+task :default => :spec

data/lib/wtf_csv/version.rb

@@ -0,0 +1,3 @@
+module WtfCSV
+  VERSION = "1.0.0"
+end

data/lib/wtf_csv/wtf_csv.rb

@@ -0,0 +1,206 @@
+module WtfCSV
+  def WtfCSV.scan(file, options = {}, &block)
+    default_options = {
+      :col_sep => ',',
+      :row_sep => $/,
+      :quote_char => '"',
+      :escape_char => '\\',
+      :check_col_count => true,
+      :col_threshold => 80,
+      :num_cols => 0,
+      :ignore_string => nil,
+      :allow_row_sep_in_quoted_fields => false,
+      :max_chars_in_field => nil,
+      :file_encoding => 'utf-8',
+    }
+    options = default_options.merge(options)
+    
+    f = File.open(file, "r:#{options[:file_encoding]}")
+    trgt_line_count = `wc -l "#{file}"`.strip.split(' ')[0].to_i if block_given?
+    
+    if options[:row_sep] == :auto
+      options[:row_sep] = SmarterCSV.guess_line_ending(f, options)
+      f.rewind
+    end
+    
+    # credit to tilo, author of smarter_csv, on how to loop over lines without reading whole file into memory
+    old_row_sep = $/
+    $/ = options[:row_sep]
+    
+    quote_errors = Array.new
+    encoding_errors = Array.new
+    column_errors = Array.new
+    column_counts = Array.new if options[:check_col_count]
+    length_errors = Array.new
+    
+    line_number = 0
+    col_number = 0
+    percent_done = 0
+    previous_line = ""
+    last_line_ended_quoted = false if options[:allow_row_sep_in_quoted_fields]
+    field_length = 0 if ! options[:max_chars_in_field].nil?
+    
+    begin
+      while ! f.eof?
+        line = f.readline
+        begin
+          if block_given? and ((line_number.to_f / trgt_line_count)*100).to_i > percent_done
+            percent_done = ((line_number.to_f / trgt_line_count)*100).to_i
+            yield percent_done
+          end
+        
+          line.chomp!
+          
+          next if ! options[:ignore_string].nil? and line == options[:ignore_string]
+          
+          if options[:allow_row_sep_in_quoted_fields] and last_line_ended_quoted
+            line_number -= 1
+            last_line_ended_quoted = false
+            field_length += options[:row_sep].length if ! options[:max_chars_in_field].nil?
+          else
+            is_quoted = false
+            new_col = true
+            quote_has_ended = false
+            quote_error = false
+            escape_char = false
+            col_number = 0
+          end
+          pos_start = 0
+          
+          line.each_char.with_index do |char, position|
+            begin
+              char.ord  # this is here to check encoding. if the encoding is bad this will throw an exception
+              
+              field_length += 1 if ! options[:max_chars_in_field].nil?
+              
+              if escape_char and options[:escape_char] == options[:quote_char] and char != options[:quote_char]
+                escape_char = false
+                is_quoted = ! is_quoted
+                if ! is_quoted
+                  quote_has_ended = true
+                elsif ! new_col
+                  quote_error = true
+                  is_quoted = false
+                end
+              end
+              
+              if char != options[:quote_char] and char != options[:col_sep] and char != options[:escape_char] ## escape_char part
+                new_col = false
+                if quote_has_ended
+                  quote_error = true
+                end
+              elsif char == options[:quote_char] and escape_char
+                escape_char = false
+              elsif char == options[:escape_char]
+                escape_char = true
+              elsif char == options[:quote_char] and is_quoted
+                quote_has_ended = true
+                is_quoted = false
+              elsif char == options[:quote_char]
+                if new_col
+                  is_quoted = true
+                  new_col = false
+                else
+                  quote_error = true
+                end
+              elsif char == options[:col_sep] and ! is_quoted
+                if quote_error
+                  quote_errors.push([line_number + 1,col_number + 1,"#{previous_line}#{line[pos_start..(position - 1)]}"])
+                  quote_error = false
+                end
+                if ! options[:max_chars_in_field].nil?
+                  length_errors.push([line_number + 1,col_number + 1,field_length - 1]) if (field_length - 1) > options[:max_chars_in_field]
+                  field_length = 0
+                end
+                new_col = true
+                quote_has_ended = false
+                previous_line = ""
+                pos_start = position + 1
+                col_number += 1
+              end
+            rescue Exception => e
+              if e.message == 'invalid byte sequence in UTF-8'
+                encoding_errors.push([line_number + 1,col_number + 1])
+              end
+            end
+          end
+          
+          if escape_char and options[:escape_char] == options[:quote_char]
+            if ! new_col and ! is_quoted
+              quote_error = true
+            else
+              is_quoted = ! is_quoted
+            end
+          end
+          
+          if is_quoted
+            if options[:allow_row_sep_in_quoted_fields]
+              last_line_ended_quoted = true
+              previous_line = "#{previous_line}#{line[pos_start...line.length]}#{options[:row_sep]}"
+              next
+            else
+              quote_error = true
+            end
+          end
+          
+          quote_errors.push([line_number + 1,col_number + 1,line[pos_start..line.length]]) if quote_error
+          
+          if ! options[:max_chars_in_field].nil?
+            length_errors.push([line_number + 1,col_number + 1,field_length]) if field_length > options[:max_chars_in_field]
+            field_length = 0
+          end
+          
+          if options[:check_col_count]
+            fnd = false
+            column_counts.each do |val|
+              if val[0] == col_number + 1
+                val[1].push(line_number)
+                fnd = true
+                break
+              end
+            end
+            
+            if ! fnd
+              column_counts.push([col_number + 1, [line_number + 1]])
+            end
+          end
+          
+        rescue Exception => e
+          # don't do anything
+        ensure
+          line_number += 1
+        end
+      end
+    ensure
+      $/ = old_row_sep
+    end
+    
+    if options[:check_col_count]
+      column_counts.sort_by! { |val| val[1].length }
+      column_counts.reverse!
+      
+      # if we're looking for an absolute number...
+      if options[:num_cols] != 0
+        column_counts.each do |val|
+          if val[0] != options[:num_cols]
+            val[1].each { |row| column_errors.push([row,val[0],options[:num_cols]]) }
+          end
+        end
+      
+      # else we'll try to figure out the target number of columns with :col_threshold
+      elsif column_counts.length > 1
+        if column_counts[0][1].length >= line_number * (options[:col_threshold].to_f / 100)
+          column_counts.drop(1).each { |val| val[1].each { |row| column_errors.push([row,val[0],column_counts[0][0]]) } }
+        else
+          column_counts.each { |val| column_errors.push([val[0],val[1].length]) }
+        end
+      end
+    end
+
+    return {quote_errors: quote_errors,
+            encoding_errors: encoding_errors,
+            column_errors: column_errors,
+            length_errors: length_errors}
+    
+  end
+end

data/lib/wtf_csv.rb

@@ -0,0 +1,3 @@
+require "wtf_csv/wtf_csv.rb"
+require "wtf_csv/version.rb"
+require 'smarter_csv'

data/spec/fixtures/check_column_count_fixed.csv

@@ -0,0 +1,5 @@
+"animal","sound",color,size
+"cat",meow,"calico",small,extra
+dog,"bark","golden",medium
+,,,
+horse,whinny,"brown"

data/spec/fixtures/check_column_count_smart_threshold.csv

@@ -0,0 +1,5 @@
+"animal","sound",color,size
+"cat",meow,"calico",small,extra
+dog,"bark","golden",medium
+,,,
+horse,whinny,"brown"

data/spec/fixtures/encoding.csv

@@ -0,0 +1,4 @@
+"animal","sound",co�lor,"\"size
+cat",meow,"calico",small_and_this_is_a_long_field_with_44_chars
+dog,"field_with_26_charact�ers"�,"golden,medium
+horse,whinny\"","brown","large"

data/spec/fixtures/escape_char.csv

@@ -0,0 +1,4 @@
+"="animal="","sound",c="olor,size
+"cat",="meow=","="calico="",small
+dog,"bark","="golden",medium="
+horse,="="whinny="=","brown","large"

data/spec/fixtures/escape_char_equals_quote_char.csv

@@ -0,0 +1,4 @@
+""animal"","""sound""","color","size"
+"""cat"",""""","meow","calico","""small""
+"dog","""bark""","""""golden""",""medium""
+"horse"," whinny ","brown",",,large"""

data/spec/fixtures/ignore_string.csv

@@ -0,0 +1,6 @@
+"animal","sound",color,size
+"cat",meow,"calico",small
+dog,"bark","golden",medium
+ignore_this_line
+do not ignore this one
+,,,

data/spec/fixtures/max_chars_in_field_n.csv

@@ -0,0 +1,4 @@
+"animal","sound",color,"\"size
+cat",meow,"calico",small_and_this_is_a_long_field_with_44_chars
+dog,"field_with_26_characters","golden,medium
+horse,whinny\"","brown","large"

data/spec/fixtures/max_chars_in_field_rn.csv

@@ -0,0 +1,4 @@
+"animal","sound",color,"\"size
+cat",meow,"calico",small_and_this_is_a_long_field_with_44_chars
+dog,"field_with_26_characters","golden,medium
+horse,whinny\"","brown","large"

data/spec/fixtures/pipe_delimited.csv

@@ -0,0 +1,4 @@
+"animal"|"sound"|color|size
+"cat"|meow|"calico"|small
+dog|"bark"|"golden"|medium
+horse|whinny|"brown"|"large"

data/spec/fixtures/quote_char.csv

@@ -0,0 +1,3 @@
++animal+,"sound",+color+,size
+++cat++,meow,+calico+,+small+
++dog+,woof,"golden",+medium+a

data/spec/fixtures/quoted_fields_n.csv

@@ -0,0 +1,4 @@
+"animal","sound",color,size
+"cat",meow,"calico",small
+dog,"bark","golden",medium
+horse,whinny,"brown","large"

data/spec/fixtures/quoted_fields_r.csv

@@ -0,0 +1 @@
+"animal","sound",color,size

data/spec/fixtures/quoted_fields_rn.csv

@@ -0,0 +1,4 @@
+"animal","sound",color,size
+"cat",meow,"calico",small
+dog,"bark","golden",medium
+horse,whinny,"brown","large"

data/spec/fixtures/quoted_newlines.csv

@@ -0,0 +1,4 @@
+"animal","sound",color,"\"size
+cat",meow,"calico",small
+dog,"bark","golden,medium
+horse,whinny","brown","large"

data/spec/spec_helper.rb

@@ -0,0 +1,12 @@
+require 'rubygems'
+require 'bundler/setup'
+
+Bundler.require(:default)
+
+require 'wtf_csv'
+
+RSpec.configure do |config|
+  config.expect_with :rspec do |c| 
+    c.syntax = :should
+  end
+end

data/spec/wtf_csv/test_spec.rb

@@ -0,0 +1,248 @@
+require 'spec_helper'
+
+fixture_path = 'spec/fixtures'
+# writing some files with special characters:
+# File.write("#{fixture_path}/quoted_fields_r.csv", "\"animal\",\"sound\",color,size\r\"cat\",meow,\"calico\",small\rdog,\"bark\",\"golden\",medium\rhorse,whinny,\"brown\",\"large\"")
+# File.write("#{fixture_path}/max_chars_in_field_rn.csv", "\"animal\",\"sound\",color,\"\\\"size\r\ncat\",meow,\"calico\",small_and_this_is_a_long_field_with_44_chars\r\ndog,\"field_with_26_characters\",\"golden,medium\r\nhorse,whinny\\\"\",\"brown\",\"large\"")
+# File.write("#{fixture_path}/encoding.csv", "\"animal\",\"sound\",co\255lor,\"\\\"size\ncat\",meow,\"calico\",small_and_this_is_a_long_field_with_44_chars\ndog,\"field_with_26_charact\255ers\"\255,\"golden,medium\nhorse,whinny\\\"\",\"brown\",\"large\"")
+
+describe 'a file with \n line endings' do
+  it 'should have no errors' do
+    options = {:row_sep => "\n"}
+    output = WtfCSV.scan("#{fixture_path}/quoted_fields_n.csv", options)
+    output[:quote_errors].length.should be == 0
+    output[:column_errors].length.should be == 0
+    output[:length_errors].length.should be == 0
+    output[:encoding_errors].length.should be == 0
+  end
+end
+
+describe 'a file with \r line endings' do
+  it 'should have no errors' do
+    options = {:row_sep => "\r"}
+    output = WtfCSV.scan("#{fixture_path}/quoted_fields_r.csv", options)
+    output[:quote_errors].length.should be == 0
+    output[:column_errors].length.should be == 0
+    output[:length_errors].length.should be == 0
+    output[:encoding_errors].length.should be == 0
+  end
+end
+
+describe 'a file with \r\n line endings' do
+  it 'should have no errors' do
+    options = {:row_sep => "\r\n"}
+    output = WtfCSV.scan("#{fixture_path}/quoted_fields_rn.csv", options)
+    output[:quote_errors].length.should be == 0
+    output[:column_errors].length.should be == 0
+    output[:length_errors].length.should be == 0
+    output[:encoding_errors].length.should be == 0
+  end
+end
+
+describe 'a file with pipe delimiters' do
+  it 'should have no errors' do
+    options = {:col_sep => '|'}
+    output = WtfCSV.scan("#{fixture_path}/pipe_delimited.csv", options)
+    output[:quote_errors].length.should be == 0
+    output[:column_errors].length.should be == 0
+    output[:length_errors].length.should be == 0
+    output[:encoding_errors].length.should be == 0
+  end
+end
+
+describe 'a file with custom quote characters' do
+  it 'should behave correctly' do
+    options = {:quote_char => '+'}
+    output = WtfCSV.scan("#{fixture_path}/quote_char.csv", options)
+    output[:quote_errors].length.should be == 2
+    if output[:quote_errors].length == 2
+      output[:quote_errors].should include [2,1,"++cat++"]
+      output[:quote_errors].should include [3,4,"+medium+a"]
+    end
+    output[:column_errors].length.should be == 0
+    output[:length_errors].length.should be == 0
+    output[:encoding_errors].length.should be == 0
+  end
+end
+
+describe 'a file with custom escape character' do
+  it 'should behave correctly' do
+    options = {:escape_char => '='}
+    output = WtfCSV.scan("#{fixture_path}/escape_char.csv", options)
+    output[:quote_errors].length.should be == 0
+    output[:column_errors].length.should be == 0
+    output[:length_errors].length.should be == 0
+    output[:encoding_errors].length.should be == 0
+  end
+end
+
+describe 'a file with the same escape and quote characters' do
+  it 'should behave correctly' do
+    options = {:escape_char => '"', :quote_char => '"'}
+    output = WtfCSV.scan("#{fixture_path}/escape_char_equals_quote_char.csv", options)
+    output[:quote_errors].length.should be == 1
+    if output[:quote_errors].length == 1
+      output[:quote_errors].should include [2,4,'"""small""']
+    end
+    output[:column_errors].length.should be == 0
+    output[:length_errors].length.should be == 0
+    output[:encoding_errors].length.should be == 0
+  end
+end
+
+describe 'checking a file for a fixed number of columns in each row' do
+  it 'should catch errors' do
+    options = {:num_cols => 4}
+    output = WtfCSV.scan("#{fixture_path}/check_column_count_fixed.csv", options)
+    output[:quote_errors].length.should be == 0
+    output[:column_errors].length.should be == 2
+    if output[:column_errors].length == 2
+      output[:column_errors].should include [2,5,4]
+      output[:column_errors].should include [5,3,4]
+    end
+    output[:length_errors].length.should be == 0
+    output[:encoding_errors].length.should be == 0
+  end
+end
+  
+describe 'checking a file for column counts and determining the number of columns with a threshold' do
+  it 'should only return the counts of how many rows have different numbers of columns if under the threshold' do
+    options = {:col_threshold => 70}
+    output = WtfCSV.scan("#{fixture_path}/check_column_count_smart_threshold.csv", options)
+    output[:quote_errors].length.should be == 0
+    output[:column_errors].length.should be == 3
+    if output[:column_errors].length == 3
+      output[:column_errors].should include [4,3]
+      output[:column_errors].should include [5,1]
+      output[:column_errors].should include [3,1]
+    end
+    output[:length_errors].length.should be == 0
+    output[:encoding_errors].length.should be == 0
+  end
+end
+
+describe 'checking a file for column counts and determining the number of columns with a threshold' do
+  it 'should determine the target number of columns using a threshold' do
+    options = {:col_threshold => 60}
+    output = WtfCSV.scan("#{fixture_path}/check_column_count_smart_threshold.csv", options)
+    output[:quote_errors].length.should be == 0
+    output[:column_errors].length.should be == 2
+    if output[:column_errors].length == 2
+      output[:column_errors].should include [2,5,4]
+      output[:column_errors].should include [5,3,4]
+    end
+    output[:length_errors].length.should be == 0
+    output[:encoding_errors].length.should be == 0
+  end
+end
+
+describe 'using the :ignore_string option' do
+  it 'should skip the line that matches this string' do
+    options = {:ignore_string => 'ignore_this_line',
+               :col_threshold => 60}
+    output = WtfCSV.scan("#{fixture_path}/ignore_string.csv", options)
+    output[:quote_errors].length.should be == 0
+    output[:column_errors].length.should be == 1
+    if output[:column_errors].length == 1
+      output[:column_errors].should include [5,1,4]
+    end
+    output[:length_errors].length.should be == 0
+    output[:encoding_errors].length.should be == 0
+  end
+end
+
+describe 'a file with quoted newlines' do
+  it 'should provide accurate errors if :allow_row_sep_in_quoted_fields is not set to true' do
+    options = {:col_threshold => 70}
+    output = WtfCSV.scan("#{fixture_path}/quoted_newlines.csv", options)
+    output[:quote_errors].length.should be == 4
+    if output[:quote_errors].length == 4
+      output[:quote_errors].should include [1,4,'"\"size']
+      output[:quote_errors].should include [2,1,'cat"']
+      output[:quote_errors].should include [3,3,'"golden,medium']
+      output[:quote_errors].should include [4,2,'whinny"']
+    end
+    output[:column_errors].length.should be == 1
+    if output[:column_errors].length == 1
+      output[:column_errors].should include [3,3,4]
+    end
+    output[:length_errors].length.should be == 0
+    output[:encoding_errors].length.should be == 0
+  end
+end
+
+describe 'a file with quoted newlines' do
+  it 'should allow the quote newlines if :allow_row_sep_in_quoted_fields is set to true' do
+    options = {:allow_row_sep_in_quoted_fields => true}
+    output = WtfCSV.scan("#{fixture_path}/quoted_newlines.csv", options)
+    output[:quote_errors].length.should be == 0
+    output[:column_errors].length.should be == 2
+    if output[:column_errors].length == 2
+      output[:column_errors].should include [7,1]
+      output[:column_errors].should include [5,1]
+    end
+    output[:length_errors].length.should be == 0
+    output[:encoding_errors].length.should be == 0
+  end
+end
+
+describe 'a file that has greater than the number of characters in :max_chars_in_field in a single field' do
+  it 'should give appropriate errors, and should count a \n in the field as a single character' do
+    options = {:max_chars_in_field => 15,
+               :allow_row_sep_in_quoted_fields => true,
+               :check_col_count => false}
+    output = WtfCSV.scan("#{fixture_path}/max_chars_in_field_n.csv", options)
+    output[:quote_errors].length.should be == 0
+    output[:column_errors].length.should be == 0
+    output[:length_errors].length.should be == 3
+    if output[:length_errors].length == 3
+      output[:length_errors].should include [1,7,44]
+      output[:length_errors].should include [2,2,26]
+      output[:length_errors].should include [2,3,30]
+    end
+    output[:encoding_errors].length.should be == 0
+  end
+end
+
+describe 'a file that has greater than the number of characters in :max_chars_in_field in a single field' do
+  it 'should give appropriate errors, and should count a \r\n in the field as two characters' do
+    options = {:max_chars_in_field => 15,
+               :allow_row_sep_in_quoted_fields => true,
+               :row_sep => "\r\n",
+               :check_col_count => false}
+    output = WtfCSV.scan("#{fixture_path}/max_chars_in_field_rn.csv", options)
+    output[:quote_errors].length.should be == 0
+    output[:column_errors].length.should be == 0
+    output[:length_errors].length.should be == 3
+    if output[:length_errors].length == 3
+      output[:length_errors].should include [1,7,44]
+      output[:length_errors].should include [2,2,26]
+      output[:length_errors].should include [2,3,31]
+    end
+    output[:encoding_errors].length.should be == 0
+  end
+end
+
+describe 'a file that has encoding errors' do
+  it 'should give appropriate errors' do
+    options = {:allow_row_sep_in_quoted_fields => true,
+               :check_col_count => false}
+    output = WtfCSV.scan("#{fixture_path}/encoding.csv", options)
+    output[:encoding_errors]
+    output[:quote_errors].length.should be == 0
+    output[:column_errors].length.should be == 0
+    output[:length_errors].length.should be == 0
+    output[:encoding_errors].length.should be == 3
+    if output[:encoding_errors].length == 3
+      output[:encoding_errors].should include [1,3]
+      output[:encoding_errors].should include [2,2]
+      output[:encoding_errors].each_with_index do |err, idx|
+        if err == [2,2]
+          output[:encoding_errors].delete_at(idx)
+          break
+        end
+      end 
+      output[:encoding_errors].should include [2,2]
+    end
+  end
+end

data/wtf_csv.gemspec

@@ -0,0 +1,21 @@
+# -*- encoding: utf-8 -*-
+require File.expand_path('../lib/wtf_csv/version', __FILE__)
+
+Gem::Specification.new do |s|
+  s.name          = 'wtf_csv'
+  s.version       = WtfCSV::VERSION
+  s.date          = '2015-09-11'
+  s.summary       = %q{Ruby gem to detect formatting issues in a CSV}
+  s.description   = %q{Ruby gem to detect formatting issues in a CSV. Can find quoting issues, incorrect column counts, and can properly handle quote-escaped line endings.}
+  s.authors       = ["Greg Merritt"]
+  s.email         = ["greg@evertrue.com"]
+  s.homepage      = 'https://github.com/gremerritt/wtf_csv'
+  s.files         = `git ls-files`.split($\)
+  s.require_paths = ["lib"]
+  s.licenses      = ['MIT']
+  
+  s.add_runtime_dependency 'smarter_csv'
+  s.add_development_dependency 'rspec'
+  
+  s.requirements << 'smarter_csv'
+end

metadata

@@ -0,0 +1,100 @@
+--- !ruby/object:Gem::Specification
+name: wtf_csv
+version: !ruby/object:Gem::Version
+  version: 1.0.0
+platform: ruby
+authors:
+- Greg Merritt
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2015-09-11 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: smarter_csv
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Ruby gem to detect formatting issues in a CSV. Can find quoting issues,
+  incorrect column counts, and can properly handle quote-escaped line endings.
+email:
+- greg@evertrue.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".rspec"
+- Gemfile
+- Gemfile.lock
+- README.md
+- Rakefile
+- lib/wtf_csv.rb
+- lib/wtf_csv/version.rb
+- lib/wtf_csv/wtf_csv.rb
+- spec/fixtures/check_column_count_fixed.csv
+- spec/fixtures/check_column_count_smart_threshold.csv
+- spec/fixtures/encoding.csv
+- spec/fixtures/escape_char.csv
+- spec/fixtures/escape_char_equals_quote_char.csv
+- spec/fixtures/ignore_string.csv
+- spec/fixtures/max_chars_in_field_n.csv
+- spec/fixtures/max_chars_in_field_rn.csv
+- spec/fixtures/pipe_delimited.csv
+- spec/fixtures/quote_char.csv
+- spec/fixtures/quoted_fields_n.csv
+- spec/fixtures/quoted_fields_r.csv
+- spec/fixtures/quoted_fields_rn.csv
+- spec/fixtures/quoted_newlines.csv
+- spec/spec_helper.rb
+- spec/wtf_csv/test_spec.rb
+- wtf_csv.gemspec
+homepage: https://github.com/gremerritt/wtf_csv
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements:
+- smarter_csv
+rubyforge_project: 
+rubygems_version: 2.0.3
+signing_key: 
+specification_version: 4
+summary: Ruby gem to detect formatting issues in a CSV
+test_files: []