csvlint 0.1.0 → 0.1.1

checksums.yaml

@@ -1,15 +1,15 @@
 ---
 !binary "U0hBMQ==":
   metadata.gz: !binary |-
-    NzY4MzE5MGI1OWI4YzkzY2U1MTZlMWJjNDM4YzM5ZjNiMWI5ZWIyMA==
+    MDgwMGE1ZmY5ZWE2MzM2ODhhMTFhZjM5NTkyZTNkZWExZjFkN2I0Mw==
   data.tar.gz: !binary |-
-    OTM5MWJmYzNkYTBiMjM4ZGU3Yjc2NTY0Njk4OTEzMDJhNmI2Yjg5Mg==
+    OTRhZTljMTc5ZDlmZDVlMmQ2ZDU2MTFmYmRlNGUyMGIyMTI3NTBmOQ==
 SHA512:
   metadata.gz: !binary |-
-    YzU4NzJmZTlkYWE1NTY0MGY4NzY0N2Q1NDYxMzI1N2YxYjZhZThmNDZhMDVj
-    Mjc1YmMyZGZkODhlNDE4OWZiYWM5NmVjNTNiOGRmMjI0OGY4ZjJkY2I4OTNl
-    ZmYwMGMwN2YwOWNlYmUzNjZjNWU3OTEzYTIwZDYzMGMzZDBiOGQ=
+    ZDE4YWVlNGYzY2E5OGJjM2ZkZjA0MzQwMWE1Yzg5YTI3YzM0MjNkMmFhNjFj
+    YmY3YzYzN2E5NDk1NDM4YmNmMGY1ZWRlNmExZWI0NmYzZmQzNTc2N2ZiOTMx
+    YzU2MjhiMmE4YTc2NTVmNzE4YWI4ZDZjOWM5MDlhNTRlY2RkMzE=
   data.tar.gz: !binary |-
-    YTkwNWVlNjlmZGZkZDJjNmUyZGQxMzBkYzViOTc1NzlhNjYzYjU3ZDljMmRl
-    YzExNDFkMDUxOThkMjJkNDRkZWYxZjNlMzdmYTRmNjI1MTliNGYzOWQxYzY3
-    MGIxMjA1YzhkYjUwOWZkOGU2NjViYjNlNjBhMTY4NTY0N2E4MmI=
+    NDY1ODhkODdlZDJlYzMyMTQ1NzFlYzAyNTYyN2YzMGE1NTM2Yzg1NWRiYTQ5
+    NmQ1ODE5MzNmOTU4ZjhmNmZjNWQwNWU1N2E5OTdmOWMyZmZiYjE4ZmVhNmVh
+    YzdmN2VmNjY1NjFkMDdmYTc0YzZiNTk2YWQxMWZkMmRkNmVhODg=

data/.ruby-version

@@ -0,0 +1 @@
+2.1.4

data/.travis.yml

@@ -1,5 +1,9 @@
 rvm:
 - 2.0.0
+- 2.1.0
+- 2.2.0
+sudo: false
+cache: bundler
 notifications:
   irc:
     channels:

data/README.md

@@ -24,12 +24,35 @@ Or install it yourself as:
 
 ## Usage
 
+You can either use this gem within your own Ruby code, or as a standolone command line application
+
+## On the command line
+
+After installing the gem, you can validate a CSV on the command line like so:
+
+	csvlint myfile.csv
+	
+You will then see the validation result, together with any warnings or errors e.g.
+
+```
+myfile.csv is INVALID
+1. blank_rows. Row: 3
+1. title_row. 
+2. inconsistent_values. Column: 14
+```
+
+You can also optionally pass a schema file like so:
+
+	csvlint myfile.csv --schema=schema.json
+
+## In your own Ruby code
+
 Currently the gem supports retrieving a CSV accessible from a URL, File, or an IO-style object (e.g. StringIO)
 
 	require 'csvlint'
 	
 	validator = Csvlint::Validator.new( "http://example.org/data.csv" )
-	validator = Csvlint::Validator.new( File.new("/path/to/my/data.csv" )
+	validator = Csvlint::Validator.new( File.new("/path/to/my/data.csv" ))
 	validator = Csvlint::Validator.new( StringIO.new( my_data_in_a_string ) )
 
 When validating from a URL the range of errors and warnings is wider as the library will also check HTTP headers for 
@@ -61,13 +84,13 @@ best practices
 ## Controlling CSV Parsing
 
 The validator supports configuration of the [CSV Dialect](http://dataprotocols.org/csv-dialect/) used in a data file. This is specified by 
-passing an options hash to the constructor:
+passing a dialect hash to the constructor:
 
-    opts = {
+    dialect = {
     	"header" => true,
     	"delimiter" => ","
     }
-	validator = Csvlint::Validator.new( "http://example.org/data.csv", opts )
+	validator = Csvlint::Validator.new( "http://example.org/data.csv", dialect )
 
 The options should be a Hash that conforms to the [CSV Dialect](http://dataprotocols.org/csv-dialect/) JSON structure.
 
@@ -205,6 +228,19 @@ Schema validation provides some additional types of error and warning messages:
 * `:below_minimum` (error) -- a column with a `minimum` constraint contains a value that is below the minimum
 * `:above_maximum` (error) -- a column with a `maximum` constraint contains a value that is above the maximum
 
+## Other validation options
+
+You can also provide an optional options hash as the fourth argument to Validator#new. Supported options are:
+
+* :limit_lines -- only check this number of lines of the CSV file. Good for a quick check on huge files.
+
+```
+options = {
+  limit_lines: 100
+}
+validator = Csvlint::Validator.new( "http://example.org/data.csv", nil, nil, options )
+```
+
 ## Contributing
 
 1. Fork it

data/bin/csvlint

@@ -3,15 +3,57 @@ $:.unshift File.join( File.dirname(__FILE__), "..", "lib")
 
 require 'csvlint'
 require 'colorize'
+require 'json'
+require 'optparse'
+require 'pp'
+
+options = {}
+opts = OptionParser.new
+
+opts.banner = "Usage: csvlint [options] [file]"
+
+opts.on("-d", "--dump-errors", "Pretty print error and warning objects.") do |d|
+  options[:dump] = d
+end
+
+opts.on("-s", "--schema-file FILENAME", "Schema file") do |s|
+  options[:schema_file] = s
+end
+
+opts.on_tail("-h", "--help",
+             "Show this message") do
+  puts opts
+  exit
+end
+
+begin
+  opts.parse!
+rescue OptionParser::InvalidOption => e
+  puts e
+  puts opts
+  exit(1)
+end
+
+def print_error(index, error, dump, color)
 
-def print_error(index, error, color=:red)
   location = ""
   location += error.row.to_s if error.row
   location += "#{error.row ? "," : ""}#{error.column.to_s}" if error.column
   if error.row || error.column
     location = "#{error.row ? "Row" : "Column"}: #{location}"
   end
-  puts "#{index+1}. #{error.type}. #{location}".colorize(color)
+  output_string = "#{index+1}. #{error.type}. #{location}"
+
+  if $stdout.tty?
+    puts output_string.colorize(color)
+  else
+    puts output_string
+  end
+
+  if dump
+    pp error
+  end
+
 end
 
 if ARGV.length == 0 && !$stdin.tty?
@@ -23,29 +65,45 @@ else
       begin
         source = File.new( source ) unless source =~ /^http(s)?/ 
       rescue Errno::ENOENT
-        puts "File not found"
+        puts "#{source} not found"
         exit 1
       end
     end
   else
-    puts "Usage: csvlint {file or URL} or {input} | csvlint"
+    puts "No CSV data to validate."
+    puts opts
     exit 1
   end
 end
 
-validator = Csvlint::Validator.new( source )
+schema = nil
+if options[:schema_file]
+  begin
+    schemafile = File.read( options[:schema_file] )
+  rescue Errno::ENOENT
+    puts "#{options[:schema_file]} not found"
+    exit 1
+  end
+  schema = Csvlint::Schema.from_json_table(nil, JSON.parse(schemafile))
+end
+
+validator = Csvlint::Validator.new( source, nil, schema )
 
-puts "#{ARGV[0] || "CSV"} is #{validator.valid? ? "VALID".green : "INVALID".red}"
+if $stdout.tty?
+  puts "#{ARGV[0] || "CSV"} is #{validator.valid? ? "VALID".green : "INVALID".red}"
+else
+  puts "#{ARGV[0] || "CSV"} is #{validator.valid? ? "VALID" : "INVALID"}"
+end
 
 if validator.errors.size > 0
   validator.errors.each_with_index do |error, i|
-    print_error(i, error)
+    print_error(i, error, options[:dump], :red)
   end
 end
 
 if validator.warnings.size > 0
   validator.warnings.each_with_index do |error, i|
-    print_error(i, error, :yellow)
+    print_error(i, error, options[:dump], :yellow)
   end
 end
 

data/features/fixtures/inconsistent-line-endings-unquoted.csv

@@ -0,0 +1,2 @@
+Foo,Bsr,Baz
+Qux,Teaspoon,Doge

data/features/parse_csv.feature

@@ -1,6 +1,6 @@
 Feature: Parse CSV
-  
-  Scenario: Sucessfully parse a valid CSV
+
+  Scenario: Successfully parse a valid CSV
     Given I have a CSV with the following content:
     """
 "Foo","Bar","Baz"
@@ -10,12 +10,12 @@ Feature: Parse CSV
     And it is stored at the url "http://example.com/example1.csv"
     When I ask if the CSV is valid
     Then I should get the value of true
-    
+
    Scenario: Successfully parse a CSV with newlines in quoted fields
     Given I have a CSV with the following content:
     """
 "a","b","c"
-"d","e","this is 
+"d","e","this is
 valid"
 "a","b","c"
 """
@@ -27,14 +27,14 @@ valid"
     Given I have a CSV with the following content:
     """
 "a","b","c"
-"d","this is 
-valid","as is this 
+"d","this is
+valid","as is this
 too"
 """
     And it is stored at the url "http://example.com/example1.csv"
     When I ask if the CSV is valid
     Then I should get the value of true
-        
+
   Scenario: Successfully report an invalid CSV
     Given I have a CSV with the following content:
     """
@@ -43,7 +43,7 @@ too"
     And it is stored at the url "http://example.com/example1.csv"
     When I ask if the CSV is valid
     Then I should get the value of false
-    
+
    Scenario: Successfully report a CSV with incorrect quoting
     Given I have a CSV with the following content:
     """
@@ -51,8 +51,8 @@ too"
     """
     And it is stored at the url "http://example.com/example1.csv"
     When I ask if the CSV is valid
-    Then I should get the value of false      
-    
+    Then I should get the value of false
+
    Scenario: Successfully report a CSV with incorrect whitespace
     Given I have a CSV with the following content:
     """
@@ -60,8 +60,8 @@ too"
     """
     And it is stored at the url "http://example.com/example1.csv"
     When I ask if the CSV is valid
-    Then I should get the value of false     
-    
+    Then I should get the value of false
+
    Scenario: Successfully report a CSV with ragged rows
     Given I have a CSV with the following content:
     """

data/features/step_definitions/information_steps.rb

@@ -4,10 +4,10 @@ end
 
 Then(/^the "(.*?)" should be "(.*?)"$/) do |type, encoding|
   validator = Csvlint::Validator.new( @url, default_csv_options ) 
-  validator.send(type.to_sym).should == encoding
+  expect( validator.send(type.to_sym) ).to eq( encoding )
 end
 
 Then(/^the metadata content type should be "(.*?)"$/) do |content_type|
   validator = Csvlint::Validator.new( @url, default_csv_options ) 
-  validator.headers['content-type'].should == content_type
+  expect( validator.headers['content-type'] ).to eq( content_type )
 end

data/features/step_definitions/parse_csv_steps.rb

@@ -1,5 +1,5 @@
 Given(/^I have a CSV with the following content:$/) do |string|
-  @csv = string
+  @csv = string.to_s
 end
 
 Given(/^it is stored at the url "(.*?)"$/) do |url|
@@ -17,7 +17,7 @@ end
 
 When(/^I ask if the CSV is valid$/) do
   @csv_options ||= default_csv_options
-  @validator = Csvlint::Validator.new( @url, @csv_options ) 
+  @validator = Csvlint::Validator.new( @url, @csv_options )
   @valid = @validator.valid?
 end
 

data/features/step_definitions/validation_errors_steps.rb

@@ -1,36 +1,36 @@
 When(/^I ask if there are errors$/) do
   @csv_options ||= default_csv_options
-  
+
   if @schema_json
     @schema = Csvlint::Schema.from_json_table( @schema_url || "http://example.org ", JSON.parse(@schema_json) )
   end
-  
-  @validator = Csvlint::Validator.new( @url, @csv_options, @schema ) 
+
+  @validator = Csvlint::Validator.new( @url, @csv_options, @schema )
   @errors = @validator.errors
 end
 
-Then(/^there should be (\d+) error$/) do |count|  
-  @errors.count.should == count.to_i
+Then(/^there should be (\d+) error$/) do |count|
+  expect( @errors.count ).to eq( count.to_i )
 end
 
 Then(/^that error should have the type "(.*?)"$/) do |type|
-  @errors.first.type.should == type.to_sym
+  expect( @errors.first.type ).to eq( type.to_sym )
 end
 
 Then(/^that error should have the row "(.*?)"$/) do |row|
-  @errors.first.row.should == row.to_i
+  expect( @errors.first.row ).to eq( row.to_i )
 end
 
 Then(/^that error should have the column "(.*?)"$/) do |column|
-  @errors.first.column.should == column.to_i
+  expect( @errors.first.column ).to eq( column.to_i )
 end
 
 Then(/^that error should have the content "(.*)"$/) do |content|
-  @errors.first.content.chomp.should == content.chomp
+  expect( @errors.first.content.chomp ).to eq( content.chomp )
 end
 
 Then(/^that error should have no content$/) do
-  @errors.first.content.should == nil
+  expect( @errors.first.content ).to eq( nil )
 end
 
 Given(/^I have a CSV that doesn't exist$/) do
@@ -40,4 +40,4 @@ end
 
 Then(/^there should be no "(.*?)" errors$/) do |type|
   @errors.each do |error| error.type.should_not == type.to_sym end
-end
+end

data/features/step_definitions/validation_info_steps.rb

@@ -10,9 +10,9 @@ Given(/^I ask if there are info messages$/) do
 end
 
 Then(/^there should be (\d+) info messages?$/) do |num|
-  @info_messages.count.should == num.to_i
+  expect( @info_messages.count ).to eq( num.to_i )
 end
 
 Then(/^one of the messages should have the type "(.*?)"$/) do |msg_type|
-  @info_messages.find{|x| x.type == msg_type.to_sym}.should be_present
+  expect( @info_messages.find{|x| x.type == msg_type.to_sym} ).to be_present
 end

data/features/step_definitions/validation_warnings_steps.rb

@@ -21,12 +21,12 @@ When(/^I ask if there are warnings$/) do
     @schema = Csvlint::Schema.from_json_table( @schema_url || "http://example.org ", JSON.parse(@schema_json) )
   end
 
-  @validator = Csvlint::Validator.new( @url, @csv_options, @schema ) 
+  @validator = Csvlint::Validator.new( @url, @csv_options, @schema )
   @warnings = @validator.warnings
 end
 
 Then(/^there should be (\d+) warnings$/) do |count|
-  @warnings.count.should == count.to_i
+  expect( @warnings.count ).to eq( count.to_i )
 end
 
 Given(/^the content type is set to "(.*?)"$/) do |type|
@@ -34,13 +34,13 @@ Given(/^the content type is set to "(.*?)"$/) do |type|
 end
 
 Then(/^that warning should have the row "(.*?)"$/) do |row|
-  @warnings.first.row.should == row.to_i
+  expect( @warnings.first.row ).to eq( row.to_i )
 end
 
 Then(/^that warning should have the column "(.*?)"$/) do |column|
-  @warnings.first.column.should == column.to_i
+  expect( @warnings.first.column ).to eq( column.to_i )
 end
 
 Then(/^that warning should have the type "(.*?)"$/) do |type|
-  @warnings.first.type.should == type.to_sym
-end
+  expect( @warnings.first.type ).to eq( type.to_sym )
+end

data/features/support/env.rb

@@ -1,17 +1,12 @@
+require 'coveralls'
+Coveralls.wear_merged!('test_frameworks')
+
 $:.unshift File.join( File.dirname(__FILE__), "..", "..", "lib")
 
-require 'simplecov'
-require 'simplecov-rcov'
 require 'rspec/expectations'
 require 'csvlint'
-require 'coveralls'
 require 'pry'
 
-Coveralls.wear_merged!
-
-SimpleCov.formatter = SimpleCov::Formatter::RcovFormatter
-SimpleCov.start
-
 require 'spork'
 
 Spork.each_run do

data/features/validation_errors.feature

@@ -148,4 +148,12 @@ Feature: Get validation errors
     And it is stored at the url "http://example.com/example1.csv"
     And I ask if there are errors
     Then there should be 1 error
-    And that error should have the type "line_breaks"
+    And that error should have the type "line_breaks"
+
+
+  Scenario: inconsistent line endings with unquoted fields in file cause an error
+    Given I have a CSV file called "inconsistent-line-endings-unquoted.csv"
+    And it is stored at the url "http://example.com/example1.csv"
+    And I ask if there are errors
+    Then there should be 1 error
+    And that error should have the type "line_breaks"

data/lib/csvlint.rb

@@ -1,9 +1,14 @@
 require 'csv'
+require 'date'
 require 'open-uri'
-require 'mime/types'
+require 'set'
 require 'tempfile'
 
-require 'csvlint/types'
+require 'active_support/core_ext/date/conversions'
+require 'active_support/core_ext/time/conversions'
+require 'mime/types'
+require 'open_uri_redirections'
+
 require 'csvlint/error_message'
 require 'csvlint/error_collector'
 require 'csvlint/validate'

data/lib/csvlint/field.rb

@@ -2,7 +2,6 @@ module Csvlint
   
   class Field
     include Csvlint::ErrorCollector
-    include Csvlint::Types
 
     attr_reader :name, :constraints, :title, :description
       
@@ -98,5 +97,73 @@ module Csvlint
         end
         return parsed
       end            
+
+      TYPE_VALIDATIONS = {
+          'http://www.w3.org/2001/XMLSchema#string'  => lambda { |value, constraints| value },
+          'http://www.w3.org/2001/XMLSchema#int'     => lambda { |value, constraints| Integer value },
+          'http://www.w3.org/2001/XMLSchema#integer' => lambda { |value, constraints| Integer value },
+          'http://www.w3.org/2001/XMLSchema#float'   => lambda { |value, constraints| Float value },
+          'http://www.w3.org/2001/XMLSchema#double'   => lambda { |value, constraints| Float value },
+          'http://www.w3.org/2001/XMLSchema#anyURI'  => lambda do |value, constraints|
+            u = URI.parse value
+            raise ArgumentError unless u.kind_of?(URI::HTTP) || u.kind_of?(URI::HTTPS)
+            u
+          end,
+          'http://www.w3.org/2001/XMLSchema#boolean' => lambda do |value, constraints|
+            return true if ['true', '1'].include? value
+            return false if ['false', '0'].include? value
+            raise ArgumentError
+          end,
+          'http://www.w3.org/2001/XMLSchema#nonPositiveInteger' => lambda do |value, constraints|
+            i = Integer value
+            raise ArgumentError unless i <= 0
+            i
+          end,
+          'http://www.w3.org/2001/XMLSchema#negativeInteger' => lambda do |value, constraints|
+            i = Integer value
+            raise ArgumentError unless i < 0
+            i
+          end,
+          'http://www.w3.org/2001/XMLSchema#nonNegativeInteger' => lambda do |value, constraints|
+            i = Integer value
+            raise ArgumentError unless i >= 0
+            i
+          end,
+          'http://www.w3.org/2001/XMLSchema#positiveInteger' => lambda do |value, constraints|
+            i = Integer value
+            raise ArgumentError unless i > 0
+            i
+          end,
+          'http://www.w3.org/2001/XMLSchema#dateTime' => lambda do |value, constraints|
+            date_pattern = constraints["datePattern"] || "%Y-%m-%dT%H:%M:%SZ"
+            d = DateTime.strptime(value, date_pattern)
+            raise ArgumentError unless d.strftime(date_pattern) == value
+            d
+          end,
+          'http://www.w3.org/2001/XMLSchema#date' => lambda do |value, constraints|
+            date_pattern = constraints["datePattern"] || "%Y-%m-%d"
+            d = Date.strptime(value, date_pattern)
+            raise ArgumentError unless d.strftime(date_pattern) == value
+            d
+          end,
+          'http://www.w3.org/2001/XMLSchema#time' => lambda do |value, constraints|
+            date_pattern = constraints["datePattern"] || "%H:%M:%S"
+            d = DateTime.strptime(value, date_pattern)
+            raise ArgumentError unless d.strftime(date_pattern) == value
+            d
+          end,
+          'http://www.w3.org/2001/XMLSchema#gYear' => lambda do |value, constraints|
+            date_pattern = constraints["datePattern"] || "%Y"
+            d = Date.strptime(value, date_pattern)
+            raise ArgumentError unless d.strftime(date_pattern) == value
+            d
+          end,
+          'http://www.w3.org/2001/XMLSchema#gYearMonth' => lambda do |value, constraints|
+            date_pattern = constraints["datePattern"] || "%Y-%m"
+            d = Date.strptime(value, date_pattern)
+            raise ArgumentError unless d.strftime(date_pattern) == value
+            d
+          end,
+      }
   end
 end

data/lib/csvlint/schema.rb

@@ -1,5 +1,3 @@
-require "set"
-
 module Csvlint
   
   class Schema

data/lib/csvlint/validate.rb

@@ -1,11 +1,8 @@
-require "open_uri_redirections"
-
 module Csvlint
   
   class Validator
     
     include Csvlint::ErrorCollector
-    include Csvlint::Types
     
     attr_reader :encoding, :content_type, :extension, :headers, :line_breaks, :dialect, :csv_header, :schema, :data
     
@@ -13,9 +10,10 @@ module Csvlint
       "Missing or stray quote" => :stray_quote,
       "Illegal quoting" => :whitespace,
       "Unclosed quoted field" => :unclosed_quote,
+      "Unquoted fields do not allow \\r or \\n" => :line_breaks,
     }
        
-    def initialize(source, dialect = nil, schema = nil)      
+    def initialize(source, dialect = nil, schema = nil, options = {})      
       @source = source
       @formats = []
       @schema = schema
@@ -31,7 +29,7 @@ module Csvlint
       }.merge(dialect || {})
             
       @csv_header = @dialect["header"]
-        
+      @limit_lines = options[:limit_lines]
       @csv_options = dialect_to_csv_options(@dialect)
       @extension = parse_extension(source)
       reset
@@ -111,19 +109,22 @@ module Csvlint
         end
         row = nil
         loop do
-         current_line = current_line + 1
+         current_line += 1
+         if @limit_lines && current_line > @limit_lines 
+           break
+         end
          begin
            wrapper.reset_line
            row = csv.shift
            @data << row
            if row             
              if current_line == 1 && header?
-               row = row.reject {|r| r.blank? }
+               row = row.reject{|col| col.nil? || col.empty?}
                validate_header(row)
                @col_counts << row.size
              else               
-               build_formats(row, current_line)
-               @col_counts << row.reject {|r| r.blank? }.size
+               build_formats(row)
+               @col_counts << row.reject{|col| col.nil? || col.empty?}.size
                @expected_columns = row.size unless @expected_columns != 0
                
                build_errors(:blank_rows, :structure, current_line, nil, wrapper.line) if row.reject{ |c| c.nil? || c.empty? }.size == 0
@@ -150,7 +151,7 @@ module Csvlint
          end
       end
       rescue ArgumentError => ae
-        build_errors(:invalid_encoding, :structure, current_line, wrapper.line) unless reported_invalid_encoding
+        build_errors(:invalid_encoding, :structure, current_line, nil, wrapper.line) unless reported_invalid_encoding
         reported_invalid_encoding = true
       end
     end          
@@ -178,7 +179,7 @@ module Csvlint
     end
     
     def fetch_error(error)
-      e = error.message.match(/^([a-z ]+) (i|o)n line ([0-9]+)\.?$/i)
+      e = error.message.match(/^(.+?)(?: [io]n)? \(?line \d+\)?\.?$/i)
       message = e[1] rescue nil
       ERROR_MATCHERS.fetch(message, :unknown_error)
     end
@@ -195,40 +196,52 @@ module Csvlint
         }
     end
     
-    def build_formats(row, line) 
+    def build_formats(row) 
       row.each_with_index do |col, i|
-        next if col.blank?
-        @formats[i] ||= []
-        
-        SIMPLE_FORMATS.each do |type, lambda|
-          begin
-            if lambda.call(col)
-              @format = type
-            end
-          rescue ArgumentError, URI::InvalidURIError
-          end
+        next if col.nil? || col.empty?
+        @formats[i] ||= Hash.new(0)
+
+        format = if col.strip[FORMATS[:numeric]]
+          :numeric
+        elsif uri?(col)
+          :uri
+        elsif col[FORMATS[:date_db]] && date_format?(Date, col, '%Y-%m-%d')
+          :date_db
+        elsif col[FORMATS[:date_short]] && date_format?(Date, col, '%e %b')
+          :date_short
+        elsif col[FORMATS[:date_rfc822]] && date_format?(Date, col, '%e %b %Y')
+          :date_rfc822
+        elsif col[FORMATS[:date_long]] && date_format?(Date, col, '%B %e, %Y')
+          :date_long
+        elsif col[FORMATS[:dateTime_time]] && date_format?(Time, col, '%H:%M')
+          :dateTime_time
+        elsif col[FORMATS[:dateTime_hms]] && date_format?(Time, col, '%H:%M:%S')
+          :dateTime_hms
+        elsif col[FORMATS[:dateTime_db]] && date_format?(Time, col, '%Y-%m-%d %H:%M:%S')
+          :dateTime_db
+        elsif col[FORMATS[:dateTime_iso8601]] && date_format?(Time, col, '%Y-%m-%dT%H:%M:%SZ')
+          :dateTime_iso8601
+        elsif col[FORMATS[:dateTime_short]] && date_format?(Time, col, '%d %b %H:%M')
+          :dateTime_short
+        elsif col[FORMATS[:dateTime_long]] && date_format?(Time, col, '%B %d, %Y %H:%M')
+          :dateTime_long
+        else
+          :string
         end
         
-        @formats[i] << @format
+        @formats[i][format] += 1
       end
     end
     
     def check_consistency
-      percentages = []
-                
-      SIMPLE_FORMATS.keys.each do |type|
-        @formats.each_with_index do |format,i|
-          percentages[i] ||= {}
-          unless format.nil?
-            percentages[i][type] = format.count(type) / format.size.to_f
+      @formats.each_with_index do |format,i|
+        if format
+          total = format.values.reduce(:+).to_f
+          if format.none?{|_,count| count / total >= 0.9}
+            build_warnings(:inconsistent_values, :schema, nil, i + 1)
           end
         end
       end
-            
-      percentages.each_with_index do |col, i|
-        next if col.values.blank?
-        build_warnings(:inconsistent_values, :schema, nil, i+1) if col.values.max < 0.9
-      end
     end
     
     private
@@ -248,6 +261,36 @@ module Csvlint
         File.extname(parsed.path)
       end
     end
-    
+
+    def uri?(value)
+      if value.strip[FORMATS[:uri]]
+        uri = URI.parse(value)
+        uri.kind_of?(URI::HTTP) || uri.kind_of?(URI::HTTPS)
+      end
+    rescue URI::InvalidURIError
+      false
+    end
+
+    def date_format?(klass, value, format)
+      klass.strptime(value, format).strftime(format) == value
+    rescue ArgumentError # invalid date
+      false
+    end
+
+    FORMATS = {
+      :string => nil,
+      :numeric => /\A[-+]?\d*\.?\d+(?:[eE][-+]?\d+)?\z/,
+      :uri => /\Ahttps?:/,
+      :date_db => /\A\d{4,}-\d\d-\d\d\z/,                                               # "12345-01-01"
+      :date_long => /\A(?:#{Date::MONTHNAMES.join('|')}) [ \d]\d, \d{4,}\z/,            # "January  1, 12345"
+      :date_rfc822 => /\A[ \d]\d (?:#{Date::ABBR_MONTHNAMES.join('|')}) \d{4,}\z/,      # " 1 Jan 12345"
+      :date_short => /\A[ \d]\d (?:#{Date::ABBR_MONTHNAMES.join('|')})\z/,              # "1 Jan"
+      :dateTime_db => /\A\d{4,}-\d\d-\d\d \d\d:\d\d:\d\d\z/,                            # "12345-01-01 00:00:00"
+      :dateTime_hms => /\A\d\d:\d\d:\d\d\z/,                                            # "00:00:00"
+      :dateTime_iso8601 => /\A\d{4,}-\d\d-\d\dT\d\d:\d\d:\d\dZ\z/,                      # "12345-01-01T00:00:00Z"
+      :dateTime_long => /\A(?:#{Date::MONTHNAMES.join('|')}) \d\d, \d{4,} \d\d:\d\d\z/, # "January 01, 12345 00:00"
+      :dateTime_short => /\A\d\d (?:#{Date::ABBR_MONTHNAMES.join('|')}) \d\d:\d\d\z/,   # "01 Jan 00:00"
+      :dateTime_time => /\A\d\d:\d\d\z/,                                                # "00:00"
+    }.freeze
   end
-end
+end

data/lib/csvlint/version.rb

@@ -1,3 +1,3 @@
 module Csvlint
-  VERSION = "0.1.0"
+  VERSION = "0.1.1"
 end

data/spec/spec_helper.rb

@@ -1,11 +1,9 @@
-require 'simplecov'
-require 'simplecov-rcov'
+require 'coveralls'
+Coveralls.wear_merged!('test_frameworks')
+
 require 'csvlint'
 require 'pry'
 require 'webmock/rspec'
-require 'coveralls'
-
-Coveralls.wear_merged!
 
 RSpec.configure do |config|
   config.treat_symbols_as_metadata_keys_with_true_values = true

data/spec/validator_spec.rb

@@ -159,21 +159,21 @@ describe Csvlint::Validator do
   context "build_formats" do
     
     {
-      "string" => "foo",
-      "numeric" => "1",
-      "uri" => "http://www.example.com",
-      "dateTime_iso8601" => "2013-01-01T13:00:00Z",
-      "date_db" => "2013-01-01",
-      "dateTime_hms" => "13:00:00"
+      :string => "foo",
+      :numeric => "1",
+      :uri => "http://www.example.com",
+      :dateTime_iso8601 => "2013-01-01T13:00:00Z",
+      :date_db => "2013-01-01",
+      :dateTime_hms => "13:00:00"
     }.each do |type, content|
       it "should return the format of #{type} correctly" do
         row = [content]
         
         validator = Csvlint::Validator.new("http://example.com/example.csv")
-        validator.build_formats(row, 1)
+        validator.build_formats(row)
         formats = validator.instance_variable_get("@formats")
         
-        formats[0].first.should == type
+        formats[0].keys.first.should == type
       end
     end
     
@@ -181,18 +181,18 @@ describe Csvlint::Validator do
       row = ["12", "3.1476"]
       
       validator = Csvlint::Validator.new("http://example.com/example.csv")
-      validator.build_formats(row, 1)
+      validator.build_formats(row)
       formats = validator.instance_variable_get("@formats")
       
-      formats[0].first.should == "numeric"
-      formats[1].first.should == "numeric"
+      formats[0].keys.first.should == :numeric
+      formats[1].keys.first.should == :numeric
     end
   
     it "should ignore blank arrays" do
       row = []
     
       validator = Csvlint::Validator.new("http://example.com/example.csv")
-      validator.build_formats(row, 1)
+      validator.build_formats(row)
       formats = validator.instance_variable_get("@formats") 
       formats.should == []
     end
@@ -207,16 +207,12 @@ describe Csvlint::Validator do
       validator = Csvlint::Validator.new("http://example.com/example.csv")
       
       rows.each_with_index do |row, i|
-        validator.build_formats(row, i)
+        validator.build_formats(row)
       end
       
       formats = validator.instance_variable_get("@formats")
       
-      formats.should == [
-          ["string",
-          "string",
-          "string"]
-        ]
+      formats.should == [{:string => 3}]
     end
   
     it "should return formats correctly if a row is blank" do
@@ -228,15 +224,15 @@ describe Csvlint::Validator do
       validator = Csvlint::Validator.new("http://example.com/example.csv")
       
       rows.each_with_index do |row, i|
-        validator.build_formats(row, i)
+        validator.build_formats(row)
       end
       
       formats = validator.instance_variable_get("@formats") 
             
       formats.should == [
-        ["string"],
-        ["numeric"],
-        ["string"]
+        {:string => 1},
+        {:numeric => 1},
+        {:string => 1},
       ]
     end
     
@@ -246,9 +242,9 @@ describe Csvlint::Validator do
     
     it "should return a warning if columns have inconsistent values" do
       formats = [
-          ["string", "string", "string"],
-          ["string", "numeric", "string"],
-          ["numeric", "numeric", "numeric"],
+          {:string => 3},
+          {:string => 2, :numeric => 1},
+          {:numeric => 3},
         ]
         
       validator = Csvlint::Validator.new("http://example.com/example.csv")
@@ -290,6 +286,17 @@ describe Csvlint::Validator do
     expect( data[2] ).to eql ['3','2','1']
   end
   
+  it "should limit number of lines read" do
+    stub_request(:get, "http://example.com/example.csv").to_return(:status => 200, 
+    :headers=>{"Content-Type" => "text/csv; header=present"}, 
+    :body => File.read(File.join(File.dirname(__FILE__),'..','features','fixtures','valid.csv')))
+    validator = Csvlint::Validator.new("http://example.com/example.csv", nil, nil, limit_lines: 2)    
+    expect( validator.valid? ).to eql(true)
+    data = validator.data
+    expect( data.count ).to eql 2    
+    expect( data[0] ).to eql ['Foo','Bar','Baz']
+  end
+  
   it "should follow redirects to SSL" do
     stub_request(:get, "http://example.com/redirect").to_return(:status => 301, :headers=>{"Location" => "https://example.com/example.csv"})
     stub_request(:get, "https://example.com/example.csv").to_return(:status => 200, 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: csvlint
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.1
 platform: ruby
 authors:
 - pezholio
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-11-27 00:00:00.000000000 Z
+date: 2015-07-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: mime-types
@@ -259,6 +259,7 @@ extra_rdoc_files: []
 files:
 - .coveralls.yml
 - .gitignore
+- .ruby-version
 - .travis.yml
 - Gemfile
 - LICENSE.md
@@ -271,6 +272,7 @@ files:
 - features/csv_options.feature
 - features/fixtures/cr-line-endings.csv
 - features/fixtures/crlf-line-endings.csv
+- features/fixtures/inconsistent-line-endings-unquoted.csv
 - features/fixtures/inconsistent-line-endings.csv
 - features/fixtures/invalid-byte-sequence.csv
 - features/fixtures/lf-line-endings.csv
@@ -300,7 +302,6 @@ files:
 - lib/csvlint/error_message.rb
 - lib/csvlint/field.rb
 - lib/csvlint/schema.rb
-- lib/csvlint/types.rb
 - lib/csvlint/validate.rb
 - lib/csvlint/version.rb
 - lib/csvlint/wrapped_io.rb
@@ -328,7 +329,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.4.2
+rubygems_version: 2.4.5
 signing_key: 
 specification_version: 4
 summary: CSV Validator
@@ -337,6 +338,7 @@ test_files:
 - features/csv_options.feature
 - features/fixtures/cr-line-endings.csv
 - features/fixtures/crlf-line-endings.csv
+- features/fixtures/inconsistent-line-endings-unquoted.csv
 - features/fixtures/inconsistent-line-endings.csv
 - features/fixtures/invalid-byte-sequence.csv
 - features/fixtures/lf-line-endings.csv

data/lib/csvlint/types.rb

@@ -1,137 +0,0 @@
-require 'set'
-require 'date'
-require 'active_support/core_ext/date/conversions'
-require 'active_support/core_ext/time/conversions'
-
-module Csvlint
-  module Types
-    SIMPLE_FORMATS = {
-      'string' => lambda { |value| true },
-      'numeric' => lambda { |value| value.strip[/\A[-+]?\d*\.?\d+(?:[eE][-+]?\d+)?\z/] },
-      'uri' => lambda do |value|
-        if value.strip[/\Ahttps?:/]
-          u = URI.parse(value)
-          u.kind_of?(URI::HTTP) || u.kind_of?(URI::HTTPS)
-        end
-      end
-    }
-
-    def self.date_format(klass, value, format, pattern)
-      if value[pattern]
-        klass.strptime(value, format).strftime(format) == value
-      end
-    end
-
-    def self.included(base)
-      [
-        [ :db, "%Y-%m-%d",
-               /\A\d{4,}-\d\d-\d\d\z/],
-        [ :number, "%Y%m%d",
-                   /\A\d{8}\z/],
-        [ :short, "%e %b",
-                  /\A[ \d]\d (?:#{Date::ABBR_MONTHNAMES.join('|')})\z/],
-        [ :rfc822, "%e %b %Y",
-                   /\A[ \d]\d (?:#{Date::ABBR_MONTHNAMES.join('|')}) \d{4,}\z/],
-        [ :long, "%B %e, %Y",
-                 /\A(?:#{Date::MONTHNAMES.join('|')}) [ \d]\d, \d{4,}\z/],
-      ].each do |type,format,pattern|
-        SIMPLE_FORMATS["date_#{type}"] = lambda do |value|
-          date_format(Date, value, format, pattern)
-        end
-      end
-
-      # strptime doesn't support widths like %9N, unlike strftime.
-      # @see http://ruby-doc.org/stdlib-2.0/libdoc/date/rdoc/DateTime.html
-      [
-        [ :time,    "%H:%M",
-                    /\A\d\d:\d\d\z/],
-        [ :hms,     "%H:%M:%S",
-                    /\A\d\d:\d\d:\d\d\z/],
-        [ :db,      "%Y-%m-%d %H:%M:%S",
-                    /\A\d{4,}-\d\d-\d\d \d\d:\d\d:\d\d\z/],
-        [ :iso8601, "%Y-%m-%dT%H:%M:%SZ",
-                    /\A\d{4,}-\d\d-\d\dT\d\d:\d\d:\d\dZ\z/],
-        [ :number,  "%Y%m%d%H%M%S",
-                    /\A\d{14}\z/],
-        [ :nsec,    "%Y%m%d%H%M%S%N",
-                    /\A\d{23}\z/],
-        [ :short,   "%d %b %H:%M",
-                    /\A\d\d (?:#{Date::ABBR_MONTHNAMES.join('|')}) \d\d:\d\d\z/],
-        [ :long,    "%B %d, %Y %H:%M",
-                    /\A(?:#{Date::MONTHNAMES.join('|')}) \d\d, \d{4,} \d\d:\d\d\z/],
-      ].each do |type,format,pattern|
-        SIMPLE_FORMATS["dateTime_#{type}"] = lambda do |value|
-          date_format(Time, value, format, pattern)
-        end
-      end
-    end
-
-    TYPE_VALIDATIONS = {
-        'http://www.w3.org/2001/XMLSchema#string'  => lambda { |value, constraints| value },
-        'http://www.w3.org/2001/XMLSchema#int'     => lambda { |value, constraints| Integer value },
-        'http://www.w3.org/2001/XMLSchema#integer' => lambda { |value, constraints| Integer value },
-        'http://www.w3.org/2001/XMLSchema#float'   => lambda { |value, constraints| Float value },
-        'http://www.w3.org/2001/XMLSchema#double'   => lambda { |value, constraints| Float value },
-        'http://www.w3.org/2001/XMLSchema#anyURI'  => lambda do |value, constraints|
-          u = URI.parse value
-          raise ArgumentError unless u.kind_of?(URI::HTTP) || u.kind_of?(URI::HTTPS)
-          u
-        end,
-        'http://www.w3.org/2001/XMLSchema#boolean' => lambda do |value, constraints|
-          return true if ['true', '1'].include? value
-          return false if ['false', '0'].include? value
-          raise ArgumentError
-        end,
-        'http://www.w3.org/2001/XMLSchema#nonPositiveInteger' => lambda do |value, constraints|
-          i = Integer value
-          raise ArgumentError unless i <= 0
-          i
-        end,
-        'http://www.w3.org/2001/XMLSchema#negativeInteger' => lambda do |value, constraints|
-          i = Integer value
-          raise ArgumentError unless i < 0
-          i
-        end,
-        'http://www.w3.org/2001/XMLSchema#nonNegativeInteger' => lambda do |value, constraints|
-          i = Integer value
-          raise ArgumentError unless i >= 0
-          i
-        end,
-        'http://www.w3.org/2001/XMLSchema#positiveInteger' => lambda do |value, constraints|
-          i = Integer value
-          raise ArgumentError unless i > 0
-          i
-        end,
-        'http://www.w3.org/2001/XMLSchema#dateTime' => lambda do |value, constraints|
-          date_pattern = constraints["datePattern"] || "%Y-%m-%dT%H:%M:%SZ"
-          d = DateTime.strptime(value, date_pattern)
-          raise ArgumentError unless d.strftime(date_pattern) == value
-          d
-        end,
-        'http://www.w3.org/2001/XMLSchema#date' => lambda do |value, constraints|
-          date_pattern = constraints["datePattern"] || "%Y-%m-%d"
-          d = Date.strptime(value, date_pattern)
-          raise ArgumentError unless d.strftime(date_pattern) == value
-          d
-        end,
-        'http://www.w3.org/2001/XMLSchema#time' => lambda do |value, constraints|
-          date_pattern = constraints["datePattern"] || "%H:%M:%S"
-          d = DateTime.strptime(value, date_pattern)
-          raise ArgumentError unless d.strftime(date_pattern) == value
-          d
-        end,
-        'http://www.w3.org/2001/XMLSchema#gYear' => lambda do |value, constraints|
-          date_pattern = constraints["datePattern"] || "%Y"
-          d = Date.strptime(value, date_pattern)
-          raise ArgumentError unless d.strftime(date_pattern) == value
-          d
-        end,
-        'http://www.w3.org/2001/XMLSchema#gYearMonth' => lambda do |value, constraints|
-          date_pattern = constraints["datePattern"] || "%Y-%m"
-          d = Date.strptime(value, date_pattern)
-          raise ArgumentError unless d.strftime(date_pattern) == value
-          d
-        end,
-    }
-  end
-end