csvlint 0.0.1

data/features/support/env.rb

@@ -0,0 +1,30 @@
+$:.unshift File.join( File.dirname(__FILE__), "..", "..", "lib")
+
+require 'simplecov'
+require 'simplecov-rcov'
+require 'rspec/expectations'
+require 'csvlint'
+require 'coveralls'
+require 'pry'
+
+Coveralls.wear_merged!
+
+SimpleCov.formatter = SimpleCov::Formatter::RcovFormatter
+SimpleCov.start
+
+require 'spork'
+
+Spork.each_run do
+  require 'csvlint'
+end
+
+class CustomWorld
+  def default_csv_options
+    return {
+    }
+  end
+end
+
+World do
+  CustomWorld.new
+end

data/features/support/webmock.rb

@@ -0,0 +1 @@
+require 'webmock/cucumber'

data/features/validation_errors.feature

@@ -0,0 +1,151 @@
+Feature: Get validation errors
+
+  Scenario: CSV with ragged rows
+    Given I have a CSV with the following content:
+    """
+"col1","col2","col3"
+"1","2","3"
+"4","5"
+    """
+    And it is stored at the url "http://example.com/example1.csv"
+    When I ask if there are errors
+    Then there should be 1 error
+    And that error should have the type "ragged_rows"
+    And that error should have the row "3"
+    And that error should have the content ""4","5""
+
+  Scenario: CSV with incorrect quoting
+    Given I have a CSV with the following content:
+    """
+"col1","col2","col3"
+"Foo","Bar","Baz
+    """
+    And it is stored at the url "http://example.com/example1.csv"
+    When I ask if there are errors
+    Then there should be 1 error
+    And that error should have the type "unclosed_quote"
+    And that error should have the row "2"
+    And that error should have the content ""Foo","Bar","Baz"
+    
+   Scenario: Successfully report a CSV with incorrect whitespace
+    Given I have a CSV with the following content:
+    """
+"col1","col2","col3"
+"Foo","Bar",   "Baz"
+    """
+    And it is stored at the url "http://example.com/example1.csv"
+    When I ask if there are errors
+    Then there should be 1 error
+    And that error should have the type "whitespace"
+    And that error should have the row "2"
+    And that error should have the content ""Foo","Bar",   "Baz""
+    
+  Scenario: Successfully report a CSV with blank rows
+    Given I have a CSV with the following content:
+    """
+"col1","col2","col3"
+"Foo","Bar","Baz"
+"","",
+"Baz","Bar","Foo"
+    """
+    And it is stored at the url "http://example.com/example1.csv"
+    When I ask if there are errors
+    Then there should be 1 error
+    And that error should have the type "blank_rows"
+    And that error should have the row "3"
+    And that error should have the content ""","","
+
+  Scenario: Successfully report a CSV with multiple trailing empty rows
+    Given I have a CSV with the following content:
+    """
+"col1","col2","col3"
+"Foo","Bar","Baz"
+"Foo","Bar","Baz"
+
+
+    """
+    And it is stored at the url "http://example.com/example1.csv"
+    When I ask if there are errors
+    Then there should be 1 error
+    And that error should have the type "blank_rows"
+    And that error should have the row "4"
+
+  Scenario: Successfully report a CSV with an empty row
+    Given I have a CSV with the following content:
+    """
+"col1","col2","col3"
+"Foo","Bar","Baz"
+
+"Foo","Bar","Baz"
+    """
+    And it is stored at the url "http://example.com/example1.csv"
+    When I ask if there are errors
+    Then there should be 1 error
+    And that error should have the type "blank_rows"
+    And that error should have the row "3"
+    
+  Scenario: Report invalid Encoding
+    Given I have a CSV file called "invalid-byte-sequence.csv"
+    And I set an encoding header of "UTF-8"
+    And it is stored at the url "http://example.com/example1.csv"
+    When I ask if there are errors
+    Then there should be 1 error    
+    And that error should have the type "invalid_encoding"
+    
+  Scenario: Correctly handle different encodings
+    Given I have a CSV file called "invalid-byte-sequence.csv"
+    And I set an encoding header of "ISO-8859-1"    
+    And it is stored at the url "http://example.com/example1.csv"
+    When I ask if there are errors
+    Then there should be no "content_encoding" errors  
+    
+  Scenario: Report invalid file
+    
+    Given I have a CSV file called "spreadsheet.xls"
+    And it is stored at the url "http://example.com/example1.csv"
+    When I ask if there are errors
+    Then there should be 1 error    
+    And that error should have the type "invalid_encoding"
+    
+  Scenario: Incorrect content type
+    Given I have a CSV with the following content:
+    """
+"abc","2","3"
+    """
+    And the content type is set to "application/excel"
+    And it is stored at the url "http://example.com/example1.xls"
+    And I ask if there are errors
+    Then there should be 1 error
+    And that error should have the type "wrong_content_type"
+
+  Scenario: Incorrect extension
+    Given I have a CSV with the following content:
+    """
+"abc","2","3"
+    """
+    And the content type is set to "application/excel"
+    And it is stored at the url "http://example.com/example1.csv"
+    And I ask if there are errors
+    Then there should be 1 error
+    And that error should have the type "wrong_content_type"
+    
+  Scenario: Handles urls that 404
+    Given I have a CSV that doesn't exist
+    When I ask if there are errors
+    Then there should be 1 error
+    And that error should have the type "not_found"
+    
+  Scenario: Incorrect line endings specified in settings
+    Given I have a CSV file called "cr-line-endings.csv"
+    And I set the line endings to linefeed
+    And it is stored at the url "http://example.com/example1.csv"
+    And I ask if there are errors
+    Then there should be 1 error
+    And that error should have the type "line_breaks"
+  
+  Scenario: inconsistent line endings in file cause an error
+    Given I have a CSV file called "inconsistent-line-endings.csv"
+    And it is stored at the url "http://example.com/example1.csv"
+    And I ask if there are errors
+    Then there should be 1 error
+    And that error should have the type "line_breaks"

data/features/validation_info.feature

@@ -0,0 +1,24 @@
+Feature: Get validation information messages
+
+  Scenario: LF line endings in file give an info message
+    Given I have a CSV file called "lf-line-endings.csv"
+    And it is stored at the url "http://example.com/example1.csv"
+    And I set header to "true" 
+    And I ask if there are info messages
+    Then there should be 1 info message
+    And that message should have the type "nonrfc_line_breaks"
+
+  Scenario: CR line endings in file give an info message
+    Given I have a CSV file called "cr-line-endings.csv"
+    And it is stored at the url "http://example.com/example1.csv"
+    And I set header to "true" 
+    And I ask if there are info messages
+    Then there should be 1 info message
+    And that message should have the type "nonrfc_line_breaks"
+
+  Scenario: CRLF line endings in file produces no info messages
+    Given I have a CSV file called "crlf-line-endings.csv"
+    And it is stored at the url "http://example.com/example1.csv"
+    And I set header to "true" 
+    And I ask if there are info messages
+    Then there should be 0 info messages

data/features/validation_warnings.feature

@@ -0,0 +1,74 @@
+Feature: Validation warnings
+
+  Scenario: UTF-8 Encoding
+    Given I have a CSV with the following content:
+    """
+"col1","col2","col3"
+"abc","2","3"
+    """
+    And it is encoded as "utf-8"
+    And it is stored at the url "http://example.com/example1.csv"
+    When I ask if there are warnings
+    Then there should be 0 warnings
+    
+   Scenario: ISO-8859-1 Encoding
+    Given I have a CSV with the following content:
+    """
+"col1","col2","col3"
+"1","2","3"
+    """
+    And it is encoded as "iso-8859-1"    
+    And it is stored at the url "http://example.com/example1.csv"
+    When I ask if there are warnings
+    Then there should be 1 warnings
+  
+  Scenario: Correct content type
+    Given I have a CSV with the following content:
+    """
+"col1","col2","col3"
+"abc","2","3"
+    """
+    And the content type is set to "text/csv"
+    And it is stored at the url "http://example.com/example1.csv"
+    And I ask if there are warnings
+    Then there should be 0 warnings
+    
+  Scenario: No extension
+    Given I have a CSV with the following content:
+    """
+"col1","col2","col3"
+"abc","2","3"
+    """
+    And the content type is set to "text/csv"
+    And it is stored at the url "http://example.com/example1"
+    And I ask if there are warnings
+    Then there should be 0 warnings
+    
+  Scenario: Allow query params after extension
+    Given I have a CSV with the following content:
+    """
+"col1","col2","col3"
+"abc","2","3"
+    """
+    And the content type is set to "text/csv"
+    And it is stored at the url "http://example.com/example1.csv?query=param"
+    And I ask if there are warnings
+    Then there should be 0 warnings
+    
+  Scenario: User doesn't supply encoding
+    Given I have a CSV with the following content:
+    """
+"col1","col2","col3"
+"abc","2","3"
+    """
+    And it is stored at the url "http://example.com/example1.csv" with no character set
+    When I ask if there are warnings
+    Then there should be 1 warnings
+    And that warning should have the type "no_encoding"
+
+  Scenario: Title rows
+    Given I have a CSV file called "title-row.csv"
+    And it is stored at the url "http://example.com/example1.csv"
+    And I ask if there are warnings
+    Then there should be 1 warnings
+    And that warning should have the type "title_row"

data/lib/csvlint.rb

@@ -0,0 +1,13 @@
+require "csvlint/version"
+require 'csv'
+require 'open-uri'
+require 'mime/types'
+require 'tempfile'
+
+require 'csvlint/types'
+require 'csvlint/error_message'
+require 'csvlint/error_collector'
+require 'csvlint/validate'
+require 'csvlint/wrapped_io'
+require 'csvlint/field'
+require 'csvlint/schema'

data/lib/csvlint/error_collector.rb

@@ -0,0 +1,43 @@
+module Csvlint
+  
+  module ErrorCollector
+      
+    def build_message(type, category, row, column, content, constraints)
+      Csvlint::ErrorMessage.new({
+                                  :type => type,
+                                  :category => category,
+                                  :row => row,
+                                  :column => column,
+                                  :content => content,
+                                  :constraints => constraints
+                                })
+    end
+    
+    MESSAGE_LEVELS = [
+      :errors,
+      :warnings,
+      :info_messages
+    ]
+    
+    MESSAGE_LEVELS.each do |level|
+      
+      attr_reader level
+      
+      define_method "build_#{level}" do |type, category = nil, row = nil, column = nil, content = nil, constraints = {}|
+        instance_variable_get("@#{level}") << build_message(type, category, row, column, content, constraints)
+      end
+      
+    end
+    
+    def valid?
+      errors.empty?
+    end
+    
+    def reset
+      MESSAGE_LEVELS.each do |level|
+        instance_variable_set("@#{level}", [])
+      end
+    end
+    
+  end
+end

data/lib/csvlint/error_message.rb

@@ -0,0 +1,15 @@
+module Csvlint
+
+  class ErrorMessage
+  
+    attr_reader :type, :category, :row, :column, :content, :constraints
+  
+    def initialize(params)
+      params.each do |key, value|
+        self.instance_variable_set("@#{key}".to_sym, value)
+      end
+    end
+  
+  end
+
+end

data/lib/csvlint/field.rb

@@ -0,0 +1,102 @@
+module Csvlint
+  
+  class Field
+    include Csvlint::ErrorCollector
+    include Csvlint::Types
+
+    attr_reader :name, :constraints, :title, :description
+      
+    def initialize(name, constraints={}, title=nil, description=nil)
+      @name = name
+      @constraints = constraints || {}
+      @uniques = Set.new
+      @title = title
+      @description = description
+      reset
+    end
+    
+    def validate_column(value, row=nil, column=nil)
+      reset
+      validate_length(value, row, column)
+      validate_values(value, row, column)
+      parsed = validate_type(value, row, column)
+      validate_range(parsed, row, column) if parsed != nil
+      return valid?
+    end
+
+    private
+      def validate_length(value, row, column)
+        if constraints["required"] == true
+          build_errors(:missing_value, :schema, row, column, value, 
+            { "required" => true }) if value.nil? || value.length == 0
+        end
+        if constraints["minLength"]
+          build_errors(:min_length, :schema, row, column, value, 
+            { "minLength" => constraints["minLength"] }) if value.nil? || value.length < constraints["minLength"]
+        end
+        if constraints["maxLength"]
+            build_errors(:max_length, :schema, row, column, value,
+             { "maxLength" => constraints["maxLength"] } ) if !value.nil? && value.length > constraints["maxLength"]
+        end
+      end
+      
+      def validate_values(value, row, column)
+        if constraints["pattern"]
+          build_errors(:pattern, :schema, row, column, value, 
+           { "pattern" => constraints["pattern"] } ) if !value.nil? && !value.match( constraints["pattern"] )
+        end
+        if constraints["unique"] == true
+          if @uniques.include? value
+            build_errors(:unique, :schema, row, column, value, { "unique" => true })
+          else
+            @uniques << value
+          end
+        end
+      end
+      
+      def validate_type(value, row, column)
+        if constraints["type"] && value != ""
+          parsed = convert_to_type(value)
+          if parsed == nil
+            failed = { "type" => constraints["type"] }
+            failed["datePattern"] = constraints["datePattern"] if constraints["datePattern"]
+            build_errors(:invalid_type, :schema, row, column, value, failed)
+            return nil
+          end
+          return parsed
+        end
+        return nil
+      end
+      
+      def validate_range(value, row, column)
+        #TODO: we're ignoring issues with converting ranges to actual types, maybe we
+        #should generate a warning? The schema is invalid
+        if constraints["minimum"]
+          minimumValue = convert_to_type( constraints["minimum"] )
+          if minimumValue
+            build_errors(:below_minimum, :schema, row, column, value, 
+              { "minimum" => constraints["minimum"] }) unless value >= minimumValue
+          end
+        end
+        if constraints["maximum"]
+          maximumValue = convert_to_type( constraints["maximum"] )
+          if maximumValue
+            build_errors(:above_maximum, :schema, row, column, value, 
+            { "maximum" => constraints["maximum"] }) unless value <= maximumValue
+          end
+        end
+      end
+
+      def convert_to_type(value)
+        parsed = nil
+        tv = TYPE_VALIDATIONS[constraints["type"]]
+        if tv
+          begin
+            parsed = tv.call value, constraints
+          rescue ArgumentError
+          end
+        end
+        return parsed
+      end            
+  end
+end