csvlint 0.1.2 → 0.1.3

checksums.yaml

@@ -1,15 +1,15 @@
 ---
 !binary "U0hBMQ==":
   metadata.gz: !binary |-
-    NjUyMGZlNGIyNjM4ZmU1OGVjMDc3Y2U2YmQ0NWEzMzM3ZGFkYzA0Yw==
+    YmFkYTMwMzU2Njg0ZDk5NmJkOTU2ZjczNWVjZTRjNTVmZWEyOTY2NA==
   data.tar.gz: !binary |-
-    NzYzMjUxY2NlZmE4ODY3NjYwNTU4NzUyMmE2NTU0NjM1MmQ0YzI4NQ==
+    ODIzOTA1MmQ0MzI3ZTZlOTc2NTU5NmFkNzgxODkxODU2MjFhY2VmYw==
 SHA512:
   metadata.gz: !binary |-
-    ODQwMGQzMjY4NGQzNGY5ZGY1Nzg3ODc5MDI3M2E3NmNmNzUwNzU1MzBjNjNk
-    YzRiMjE3NTJlNTc5YWI2NjhkNGJjNDcxNzVkMDgyZmRkYjZlYWFmZjUyOWQx
-    ZmVkOWRjZDJhMGU4M2QyYmI0YzZmMDBlYWRhMjFmNGFlZmJmZTg=
+    Y2IwYTYzYjg1N2JmMmFmN2Q1NjIwYTQxMTU0Y2Y5YTM3YzFkYjkyZWI1MWEy
+    ODJiZTZjNDhlMDc0MjliZGRkMDQyZDRkMDdlNzRjMmFhN2IyYTJkZDU4MWZl
+    NzdmYWQwZDI5MTI1Nzk5ZDQxZjRiZGVlMTI3NzMwNmI2NGZiZTM=
   data.tar.gz: !binary |-
-    ZjhkODBhMWU1ODFjN2JiNjQ3ZjBlZmFkNzAwMmVhOTNiYzgwNDNjM2RjODg5
-    YWE2NGUwNjk1M2JlYWFjZmVlY2U0YjNmMTZmM2U4YTFlMDcyMGViNWU3NWY2
-    NTY5NmIyNmE5ZjYwNDkwOTE1NjQ2MWRiZGU0ZGRhM2MxMTEyYjU=
+    NDBlNGVhMTVkM2IzNDIyNWEzMjU5NDI4YjZjNjlmODg5YWNiOTMzNDk4ZGQ5
+    ODVmMTM2MWJjNTIwNzdhNmNkNTNhZGUyZTUyZTgyYzM4Njc4YWUwYzA1YWJl
+    MmYwNmFkMTVlMGIyNDE5ODVkN2E3NWE5ZWFlYzQ1NGVjNzY1Y2Q=

data/CHANGELOG.md

@@ -0,0 +1,137 @@
+# Change Log
+
+## [Unreleased](https://github.com/theodi/csvlint.rb/tree/HEAD)
+
+[Full Changelog](https://github.com/theodi/csvlint.rb/compare/0.1.2...HEAD)
+
+**Merged pull requests:**
+
+- Error reporting schema expanded test suite [\#138](https://github.com/theodi/csvlint.rb/pull/138) ([quadrophobiac](https://github.com/quadrophobiac))
+
+- Validate header size improvement [\#137](https://github.com/theodi/csvlint.rb/pull/137) ([adamc00](https://github.com/adamc00))
+
+- Invalid schema [\#132](https://github.com/theodi/csvlint.rb/pull/132) ([bcouston](https://github.com/bcouston))
+
+## [0.1.2](https://github.com/theodi/csvlint.rb/tree/0.1.2) (2015-07-15)
+
+[Full Changelog](https://github.com/theodi/csvlint.rb/compare/0.1.1...0.1.2)
+
+**Closed issues:**
+
+- When an encoding error is thrown the line content is put into the column field in the error object [\#131](https://github.com/theodi/csvlint.rb/issues/131)
+
+**Merged pull requests:**
+
+- Catch invalid URIs [\#133](https://github.com/theodi/csvlint.rb/pull/133) ([pezholio](https://github.com/pezholio))
+
+- Emit a warning when the CSV header does not match the supplied schema [\#127](https://github.com/theodi/csvlint.rb/pull/127) ([adamc00](https://github.com/adamc00))
+
+## [0.1.1](https://github.com/theodi/csvlint.rb/tree/0.1.1) (2015-07-13)
+
+[Full Changelog](https://github.com/theodi/csvlint.rb/compare/0.1.0...0.1.1)
+
+**Closed issues:**
+
+- Add Command Line Support [\#128](https://github.com/theodi/csvlint.rb/issues/128)
+
+- BUG: Incorrect inconsistent\_values error on numeric columns [\#106](https://github.com/theodi/csvlint.rb/issues/106)
+
+**Merged pull requests:**
+
+- Fixes line content incorrectly being put into the row column field when there is an encoding error. [\#130](https://github.com/theodi/csvlint.rb/pull/130) ([glacier](https://github.com/glacier))
+
+- Add command line help [\#129](https://github.com/theodi/csvlint.rb/pull/129) ([pezholio](https://github.com/pezholio))
+
+- Remove stray q character. [\#125](https://github.com/theodi/csvlint.rb/pull/125) ([adamc00](https://github.com/adamc00))
+
+- csvlint utility can take arguments to specify a schema and pp errors [\#124](https://github.com/theodi/csvlint.rb/pull/124) ([adamc00](https://github.com/adamc00))
+
+- Fixed warning - use expect\( \) rather than .should [\#123](https://github.com/theodi/csvlint.rb/pull/123) ([jezhiggins](https://github.com/jezhiggins))
+
+- Fixed spelling mistake [\#121](https://github.com/theodi/csvlint.rb/pull/121) ([jezhiggins](https://github.com/jezhiggins))
+
+- Avoid using \#blank? if unnecessary [\#120](https://github.com/theodi/csvlint.rb/pull/120) ([jpmckinney](https://github.com/jpmckinney))
+
+- eliminate some date and time formats, related \#105 [\#119](https://github.com/theodi/csvlint.rb/pull/119) ([jpmckinney](https://github.com/jpmckinney))
+
+- Match another CSV error about line endings [\#118](https://github.com/theodi/csvlint.rb/pull/118) ([jpmckinney](https://github.com/jpmckinney))
+
+- fixed typo mistake in README [\#117](https://github.com/theodi/csvlint.rb/pull/117) ([railsfactory-kumaresan](https://github.com/railsfactory-kumaresan))
+
+- Integrate @jpmickinney's build\_formats improvements [\#112](https://github.com/theodi/csvlint.rb/pull/112) ([Floppy](https://github.com/Floppy))
+
+- make limit\_lines into a non-dialect option [\#110](https://github.com/theodi/csvlint.rb/pull/110) ([Floppy](https://github.com/Floppy))
+
+- fix coveralls stats [\#109](https://github.com/theodi/csvlint.rb/pull/109) ([Floppy](https://github.com/Floppy))
+
+- Speed up \#build\_formats \(changes its API\) [\#103](https://github.com/theodi/csvlint.rb/pull/103) ([jpmckinney](https://github.com/jpmckinney))
+
+- Limit lines [\#101](https://github.com/theodi/csvlint.rb/pull/101) ([Hoedic](https://github.com/Hoedic))
+
+## [0.1.0](https://github.com/theodi/csvlint.rb/tree/0.1.0) (2014-11-27)
+
+**Implemented enhancements:**
+
+- Blank values shouldn't count as inconsistencies [\#90](https://github.com/theodi/csvlint.rb/issues/90)
+
+- Make sure we don't check schema column count and ragged row count together [\#66](https://github.com/theodi/csvlint.rb/issues/66)
+
+- Include the failed constraints in error message when doing field validation [\#64](https://github.com/theodi/csvlint.rb/issues/64)
+
+- Include the column value in error message when field validation fails [\#63](https://github.com/theodi/csvlint.rb/issues/63)
+
+- Expose optional JSON table schema fields [\#55](https://github.com/theodi/csvlint.rb/issues/55)
+
+- Ensure header rows are properly handled and validated [\#48](https://github.com/theodi/csvlint.rb/issues/48)
+
+- Support zipped CSV? [\#30](https://github.com/theodi/csvlint.rb/issues/30)
+
+- Improve feedback on inconsistent values [\#29](https://github.com/theodi/csvlint.rb/issues/29)
+
+- Reported error positions are not massively useful [\#15](https://github.com/theodi/csvlint.rb/issues/15)
+
+**Fixed bugs:**
+
+- undefined method `\[\]' for nil:NilClass from fetch\_error [\#71](https://github.com/theodi/csvlint.rb/issues/71)
+
+- Inconsistent column bases [\#69](https://github.com/theodi/csvlint.rb/issues/69)
+
+- Improve error handling in Schema loading [\#42](https://github.com/theodi/csvlint.rb/issues/42)
+
+- Recover from some line ending problems [\#41](https://github.com/theodi/csvlint.rb/issues/41)
+
+- Inconsistent values due to number format differences [\#32](https://github.com/theodi/csvlint.rb/issues/32)
+
+- New lines in quoted fields are valid [\#31](https://github.com/theodi/csvlint.rb/issues/31)
+
+- Wrongly reporting incorrect file extension [\#23](https://github.com/theodi/csvlint.rb/issues/23)
+
+- Incorrect extension reported when URL has query options at the end [\#14](https://github.com/theodi/csvlint.rb/issues/14)
+
+**Closed issues:**
+
+- Get gem continuously deploying [\#93](https://github.com/theodi/csvlint.rb/issues/93)
+
+- Publish on rubygems.org [\#92](https://github.com/theodi/csvlint.rb/issues/92)
+
+- Duplicate column names [\#87](https://github.com/theodi/csvlint.rb/issues/87)
+
+- Return code is always 0 \(except when it isn't\) [\#85](https://github.com/theodi/csvlint.rb/issues/85)
+
+- Can't pipe data to csvlint [\#84](https://github.com/theodi/csvlint.rb/issues/84)
+
+- They have some validator running if someone wants to inspect it for "inspiration" [\#27](https://github.com/theodi/csvlint.rb/issues/27)
+
+- Allow CSV parsing options to be configured as a parameter [\#6](https://github.com/theodi/csvlint.rb/issues/6)
+
+- Use explicit CSV parsing options [\#5](https://github.com/theodi/csvlint.rb/issues/5)
+
+- Improving encoding detection [\#2](https://github.com/theodi/csvlint.rb/issues/2)
+
+**Merged pull requests:**
+
+- Continuously deploy gem [\#102](https://github.com/theodi/csvlint.rb/pull/102) ([pezholio](https://github.com/pezholio))
+
+
+
+\* *This Change Log was automatically generated by [github_changelog_generator](https://github.com/skywinder/Github-Changelog-Generator)*

data/csvlint.gemspec

@@ -36,4 +36,5 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "rspec-expectations"
   spec.add_development_dependency "coveralls"
   spec.add_development_dependency "pry"
+  spec.add_development_dependency "github_changelog_generator"
 end

data/features/csvupload.feature

@@ -0,0 +1,153 @@
+Feature: Collect all the tests that should trigger dialect check related errors
+
+  Scenario: Title rows, I wish to trigger a :title_row type message
+    Given I have a CSV file called "title-row.csv"
+    And it is stored at the url "http://example.com/example1.csv"
+    And I ask if there are warnings
+    Then there should be 1 warnings
+    And that warning should have the type "title_row"
+
+#    :nonrfc_line_breaks
+
+  Scenario: LF line endings in file give an info message of type :nonrfc_line_breaks
+    Given I have a CSV file called "lf-line-endings.csv"
+    And it is stored at the url "http://example.com/example1.csv"
+    And I set header to "true"
+    And I ask if there are info messages
+    Then there should be 2 info messages
+    And one of the messages should have the type "nonrfc_line_breaks"
+
+  Scenario: CR line endings in file give an info message of type :nonrfc_line_breaks
+    Given I have a CSV file called "cr-line-endings.csv"
+    And it is stored at the url "http://example.com/example1.csv"
+    And I set header to "true"
+    And I ask if there are info messages
+    Then there should be 2 info messages
+    And one of the messages should have the type "nonrfc_line_breaks"
+
+  Scenario: CRLF line endings in file produces no info messages of type :nonrfc_line_breaks
+    Given I have a CSV file called "crlf-line-endings.csv"
+    And it is stored at the url "http://example.com/example1.csv"
+    And I set header to "true"
+    And I ask if there are info messages
+    Then there should be 1 info message
+
+#  :line_breaks
+
+  Scenario: Incorrect line endings specified in settings
+    Given I have a CSV file called "cr-line-endings.csv"
+    And I set the line endings to linefeed
+    And it is stored at the url "http://example.com/example1.csv"
+    And I ask if there are errors
+    Then there should be 1 error
+    And that error should have the type "line_breaks"
+
+  Scenario: inconsistent line endings in file cause an error
+    Given I have a CSV file called "inconsistent-line-endings.csv"
+    And it is stored at the url "http://example.com/example1.csv"
+    And I ask if there are errors
+    Then there should be 1 error
+    And that error should have the type "line_breaks"
+
+
+  Scenario: inconsistent line endings with unquoted fields in file cause an error
+    Given I have a CSV file called "inconsistent-line-endings-unquoted.csv"
+    And it is stored at the url "http://example.com/example1.csv"
+    And I ask if there are errors
+    Then there should be 1 error
+    And that error should have the type "line_breaks"
+
+#:unclosed_quote
+
+  Scenario: CSV with incorrect quoting
+    Given I have a CSV with the following content:
+    """
+"col1","col2","col3"
+"Foo","Bar","Baz
+    """
+    And it is stored at the url "http://example.com/example1.csv"
+    When I ask if there are errors
+    Then there should be 1 error
+    And that error should have the type "unclosed_quote"
+    And that error should have the row "2"
+    And that error should have the content ""Foo","Bar","Baz"
+
+#  :invalid_encoding
+
+  Scenario: Report invalid Encoding
+    Given I have a CSV file called "invalid-byte-sequence.csv"
+    And I set an encoding header of "UTF-8"
+    And it is stored at the url "http://example.com/example1.csv"
+    When I ask if there are errors
+    Then there should be 1 error
+    And that error should have the type "invalid_encoding"
+
+  Scenario: Report invalid file
+#should this throw an excel error?
+    Given I have a CSV file called "spreadsheet.xls"
+    And it is stored at the url "http://example.com/example1.csv"
+    When I ask if there are errors
+    Then there should be 1 error
+    And that error should have the type "invalid_encoding"
+
+#  :blank_rows
+
+  Scenario: Successfully report a CSV with blank rows
+    Given I have a CSV with the following content:
+    """
+"col1","col2","col3"
+"Foo","Bar","Baz"
+"","",
+"Baz","Bar","Foo"
+    """
+    And it is stored at the url "http://example.com/example1.csv"
+    When I ask if there are errors
+    Then there should be 1 error
+    And that error should have the type "blank_rows"
+    And that error should have the row "3"
+    And that error should have the content ""","","
+
+  Scenario: Successfully report a CSV with multiple trailing empty rows
+    Given I have a CSV with the following content:
+    """
+"col1","col2","col3"
+"Foo","Bar","Baz"
+"Foo","Bar","Baz"
+
+
+    """
+    And it is stored at the url "http://example.com/example1.csv"
+    When I ask if there are errors
+    Then there should be 1 error
+    And that error should have the type "blank_rows"
+    And that error should have the row "4"
+
+  Scenario: Successfully report a CSV with an empty row
+    Given I have a CSV with the following content:
+    """
+"col1","col2","col3"
+"Foo","Bar","Baz"
+
+"Foo","Bar","Baz"
+    """
+    And it is stored at the url "http://example.com/example1.csv"
+    When I ask if there are errors
+    Then there should be 1 error
+    And that error should have the type "blank_rows"
+    And that error should have the row "3"
+
+#:check_options
+
+  Scenario: Warn if options seem to return invalid data
+    Given I have a CSV with the following content:
+    """
+'Foo';'Bar';'Baz'
+'1';'2';'3'
+'3';'2';'1'
+    """
+    And I set the delimiter to ","
+    And I set quotechar to """
+    And it is stored at the url "http://example.com/example1.csv"
+    And I ask if there are warnings
+    Then there should be 1 warnings
+    And that warning should have the type "check_options"

data/features/schema_validation.feature

@@ -60,4 +60,46 @@ Feature: Schema Validation
     """
     When I ask if there are warnings
     Then there should be 1 warnings
-    
+
+  Scenario: Schema with valid regex
+    Given I have a CSV with the following content:
+    """
+  "firstname","id","email"
+  "Bob","1234","bob@example.org"
+  "Alice","5","alice@example.com"
+    """
+    And it is stored at the url "http://example.com/example1.csv"
+    And I have a schema with the following content:
+    """
+{
+  "fields": [
+          { "name": "Name", "constraints": { "required": true, "pattern": "^[A-Za-z0-9_]*$" } },
+          { "name": "Id", "constraints": { "required": true, "minLength": 1 } },
+          { "name": "Email", "constraints": { "required": true } }
+    ]
+}
+    """
+    When I ask if there are errors
+    Then there should be 0 error
+
+  Scenario: Schema with invalid regex
+    Given I have a CSV with the following content:
+    """
+  "firstname","id","email"
+  "Bob","1234","bob@example.org"
+  "Alice","5","alice@example.com"
+    """
+    And it is stored at the url "http://example.com/example1.csv"
+    And I have a schema with the following content:
+    """
+{
+  "fields": [
+          { "name": "Name", "constraints": { "required": true, "pattern": "((" } },
+          { "name": "Id", "constraints": { "required": true, "minLength": 1 } },
+          { "name": "Email", "constraints": { "required": true } }
+    ]
+}
+    """
+    When I ask if there are errors
+    Then there should be 1 error
+    And that error should have the type "invalid_regex"

data/lib/csvlint/error_collector.rb

@@ -1,13 +1,15 @@
 module Csvlint
   module ErrorCollector
     attr_reader :errors, :warnings, :info_messages
-
+    # Creates a validation error
     def build_errors(type, category = nil, row = nil, column = nil, content = nil, constraints = {})
       @errors << Csvlint::ErrorMessage.new(type, category, row, column, content, constraints)
     end
+    # Creates a validation warning
     def build_warnings(type, category = nil, row = nil, column = nil, content = nil, constraints = {})
       @warnings << Csvlint::ErrorMessage.new(type, category, row, column, content, constraints)
     end
+    # Creates a validation information message
     def build_info_messages(type, category = nil, row = nil, column = nil, content = nil, constraints = {})
       @info_messages << Csvlint::ErrorMessage.new(type, category, row, column, content, constraints)
     end

data/lib/csvlint/error_message.rb

@@ -9,6 +9,7 @@ module Csvlint
       @column = column
       @content = content
       @constraints = constraints
+
     end
   end
 end

data/lib/csvlint/field.rb

@@ -1,10 +1,10 @@
 module Csvlint
-  
+
   class Field
     include Csvlint::ErrorCollector
 
     attr_reader :name, :constraints, :title, :description
-      
+
     def initialize(name, constraints={}, title=nil, description=nil)
       @name = name
       @constraints = constraints || {}
@@ -13,9 +13,12 @@ module Csvlint
       @description = description
       reset
     end
-    
-    def validate_column(value, row=nil, column=nil)
+
+    def validate_column(value, row=nil, column=nil, all_errors=[])
       reset
+      unless all_errors.any?{|error| ((error.type == :invalid_regex) && (error.column == column))}
+        validate_regex(value, row, column)
+      end
       validate_length(value, row, column)
       validate_values(value, row, column)
       parsed = validate_type(value, row, column)
@@ -26,11 +29,11 @@ module Csvlint
     private
       def validate_length(value, row, column)
         if constraints["required"] == true
-          build_errors(:missing_value, :schema, row, column, value, 
+          build_errors(:missing_value, :schema, row, column, value,
             { "required" => true }) if value.nil? || value.length == 0
         end
         if constraints["minLength"]
-          build_errors(:min_length, :schema, row, column, value, 
+          build_errors(:min_length, :schema, row, column, value,
             { "minLength" => constraints["minLength"] }) if value.nil? || value.length < constraints["minLength"]
         end
         if constraints["maxLength"]
@@ -38,12 +41,26 @@ module Csvlint
              { "maxLength" => constraints["maxLength"] } ) if !value.nil? && value.length > constraints["maxLength"]
         end
       end
-      
-      def validate_values(value, row, column)
-        if constraints["pattern"]
-          build_errors(:pattern, :schema, row, column, value, 
-           { "pattern" => constraints["pattern"] } ) if !value.nil? && !value.match( constraints["pattern"] )
+
+      def validate_regex(value, row, column)
+        pattern = constraints["pattern"]
+        if pattern
+          begin
+            Regexp.new(pattern)
+            build_errors(:pattern, :schema, row, column, value,
+            { "pattern" => constraints["pattern"] } ) if !value.nil? && !value.match( constraints["pattern"] )
+          rescue RegexpError
+            build_errors(:invalid_regex, :schema, nil, column, ("#{name}: Constraints: Pattern: #{pattern}"),
+              { "pattern" => constraints["pattern"] })
+          end
         end
+      end
+
+      def validate_values(value, row, column)
+        # If a pattern exists, raise an invalid regex error if it is not in
+        # valid regex form, else, if the value of the relevant field in the csv
+        # does not match the given regex pattern in the schema, raise a
+        # pattern error.
         if constraints["unique"] == true
           if @uniques.include? value
             build_errors(:unique, :schema, row, column, value, { "unique" => true })
@@ -52,7 +69,7 @@ module Csvlint
           end
         end
       end
-      
+
       def validate_type(value, row, column)
         if constraints["type"] && value != ""
           parsed = convert_to_type(value)
@@ -66,21 +83,21 @@ module Csvlint
         end
         return nil
       end
-      
+
       def validate_range(value, row, column)
         #TODO: we're ignoring issues with converting ranges to actual types, maybe we
         #should generate a warning? The schema is invalid
         if constraints["minimum"]
           minimumValue = convert_to_type( constraints["minimum"] )
           if minimumValue
-            build_errors(:below_minimum, :schema, row, column, value, 
+            build_errors(:below_minimum, :schema, row, column, value,
               { "minimum" => constraints["minimum"] }) unless value >= minimumValue
           end
         end
         if constraints["maximum"]
           maximumValue = convert_to_type( constraints["maximum"] )
           if maximumValue
-            build_errors(:above_maximum, :schema, row, column, value, 
+            build_errors(:above_maximum, :schema, row, column, value,
             { "maximum" => constraints["maximum"] }) unless value <= maximumValue
           end
         end
@@ -96,7 +113,7 @@ module Csvlint
           end
         end
         return parsed
-      end            
+      end
 
       TYPE_VALIDATIONS = {
           'http://www.w3.org/2001/XMLSchema#string'  => lambda { |value, constraints| value },
@@ -170,4 +187,4 @@ module Csvlint
           end,
       }
   end
-end
+end

data/lib/csvlint/schema.rb

@@ -1,11 +1,11 @@
 module Csvlint
-  
+
   class Schema
-    
+
     include Csvlint::ErrorCollector
-    
+
     attr_reader :uri, :fields, :title, :description
-    
+
     def initialize(uri, fields=[], title=nil, description=nil)
       @uri = uri
       @fields = fields
@@ -17,16 +17,16 @@ module Csvlint
     def validate_header(header)
       reset
 
-      found_header = header.join(',')
-      expected_header = @fields.map{ |f| f.name }.join(',')
+      found_header = header.to_csv(:row_sep => '')
+      expected_header = @fields.map{ |f| f.name }.to_csv(:row_sep => '')
       if found_header != expected_header
         build_warnings(:malformed_header, :schema, 1, nil, found_header, expected_header)
       end
 
       return valid?
     end
-        
-    def validate_row(values, row=nil)
+
+    def validate_row(values, row=nil, all_errors=[])
       reset
       if values.length < fields.length
         fields[values.size..-1].each_with_index do |field, i|
@@ -38,34 +38,37 @@ module Csvlint
           build_warnings(:extra_column, :schema, row, fields.size+i+1)
         end
       end
-      
+
       fields.each_with_index do |field,i|
         value = values[i] || ""
-        result = field.validate_column(value, row, i+1)
+        result = field.validate_column(value, row, i+1, all_errors)
         @errors += fields[i].errors
-        @warnings += fields[i].warnings        
+        @warnings += fields[i].warnings
       end
-            
+
       return valid?
     end
-    
+
     def Schema.from_json_table(uri, json)
       fields = []
       json["fields"].each do |field_desc|
-        fields << Csvlint::Field.new( field_desc["name"] , field_desc["constraints"], 
+        fields << Csvlint::Field.new( field_desc["name"] , field_desc["constraints"],
           field_desc["title"], field_desc["description"] )
       end if json["fields"]
       return Schema.new( uri , fields, json["title"], json["description"] )
     end
-    
+
+    # Difference in functionality between from_json_table and load_from_json_table
+    # needs to be specified
+
     def Schema.load_from_json_table(uri)
       begin
         json = JSON.parse( open(uri).read )
         return Schema.from_json_table(uri,json)
       rescue
-        return nil
+        return Schema.new(nil, [], "malformed", "malformed")
       end
     end
-    
+
   end
-end
+end

data/lib/csvlint/validate.rb

@@ -1,25 +1,26 @@
 module Csvlint
-  
+
   class Validator
-    
+
     include Csvlint::ErrorCollector
-    
+
     attr_reader :encoding, :content_type, :extension, :headers, :line_breaks, :dialect, :csv_header, :schema, :data
-    
+
     ERROR_MATCHERS = {
       "Missing or stray quote" => :stray_quote,
       "Illegal quoting" => :whitespace,
       "Unclosed quoted field" => :unclosed_quote,
       "Unquoted fields do not allow \\r or \\n" => :line_breaks,
     }
-       
-    def initialize(source, dialect = nil, schema = nil, options = {})      
+
+    def initialize(source, dialect = nil, schema = nil, options = {})
+
       @source = source
       @formats = []
       @schema = schema
-      
+
       @supplied_dialect = dialect != nil
-            
+
       @dialect = {
         "header" => true,
         "delimiter" => ",",
@@ -27,18 +28,19 @@ module Csvlint
         "lineTerminator" => :auto,
         "quoteChar" => '"'
       }.merge(dialect || {})
-            
+
       @csv_header = @dialect["header"]
       @limit_lines = options[:limit_lines]
       @csv_options = dialect_to_csv_options(@dialect)
-      @extension = parse_extension(source)
+      @extension = parse_extension(source) unless @source.nil?
       reset
       validate
+
     end
-        
+
     def validate
-      single_col = false   
-      io = nil   
+      single_col = false
+      io = nil
       begin
         io = @source.respond_to?(:gets) ? @source : open(@source, :allow_redirections=>:all)
         validate_metadata(io)
@@ -47,19 +49,19 @@ module Csvlint
         unless sum.nil?
           build_warnings(:title_row, :structure) if @col_counts.first < (sum / @col_counts.size.to_f)
         end
-        build_warnings(:check_options, :structure) if @expected_columns == 1        
-        check_consistency      
+        build_warnings(:check_options, :structure) if @expected_columns == 1
+        check_consistency
       rescue OpenURI::HTTPError, Errno::ENOENT
         build_errors(:not_found)
       ensure
         io.close if io && io.respond_to?(:close)
       end
     end
-    
+
     def validate_metadata(io)
       @encoding = io.charset rescue nil
       @content_type = io.content_type rescue nil
-      @headers = io.meta rescue nil    
+      @headers = io.meta rescue nil
       assumed_header = undeclared_header = !@supplied_dialect
       if @headers
         if @headers["content-type"] =~ /text\/csv/
@@ -74,31 +76,33 @@ module Csvlint
           assumed_header = false
         end
         if @headers["content-type"] !~ /charset=/
-          build_warnings(:no_encoding, :context) 
+          build_warnings(:no_encoding, :context)
         else
           build_warnings(:encoding, :context) if @encoding != "utf-8"
         end
         build_warnings(:no_content_type, :context) if @content_type == nil
         build_warnings(:excel, :context) if @content_type == nil && @extension =~ /.xls(x)?/
         build_errors(:wrong_content_type, :context) unless (@content_type && @content_type =~ /text\/csv/)
-        
+
         if undeclared_header
           build_errors(:undeclared_header, :structure)
           assumed_header = false
         end
-        
+
       end
       build_info_messages(:assumed_header, :structure) if assumed_header
     end
-    
+
+    # analyses the provided csv and builds errors, warnings and info messages
     def parse_csv(io)
       @expected_columns = 0
       current_line = 0
       reported_invalid_encoding = false
+      all_errors = []
       @col_counts = []
-      
-      @csv_options[:encoding] = @encoding  
-  
+
+      @csv_options[:encoding] = @encoding
+
       begin
         wrapper = WrappedIO.new( io )
         csv = CSV.new( wrapper, @csv_options )
@@ -110,37 +114,38 @@ module Csvlint
         row = nil
         loop do
          current_line += 1
-         if @limit_lines && current_line > @limit_lines 
+         if @limit_lines && current_line > @limit_lines
            break
          end
          begin
            wrapper.reset_line
            row = csv.shift
            @data << row
-           if row             
+           if row
              if current_line == 1 && header?
                row = row.reject{|col| col.nil? || col.empty?}
                validate_header(row)
                @col_counts << row.size
-             else               
+             else
                build_formats(row)
                @col_counts << row.reject{|col| col.nil? || col.empty?}.size
                @expected_columns = row.size unless @expected_columns != 0
-               
+
                build_errors(:blank_rows, :structure, current_line, nil, wrapper.line) if row.reject{ |c| c.nil? || c.empty? }.size == 0
-               
+               # Builds errors and warnings related to the provided schema file
                if @schema
-                 @schema.validate_row(row, current_line)
+                 @schema.validate_row(row, current_line, all_errors)
                  @errors += @schema.errors
+                 all_errors += @schema.errors
                  @warnings += @schema.warnings
                else
                  build_errors(:ragged_rows, :structure, current_line, nil, wrapper.line) if !row.empty? && row.size != @expected_columns
                end
-               
+
              end
-           else             
+           else
              break
-           end         
+           end
          rescue CSV::MalformedCSVError => e
            type = fetch_error(e)
            if type == :stray_quote && !wrapper.line.match(csv.row_sep)
@@ -154,8 +159,8 @@ module Csvlint
         build_errors(:invalid_encoding, :structure, current_line, nil, wrapper.line) unless reported_invalid_encoding
         reported_invalid_encoding = true
       end
-    end          
-    
+    end
+
     def validate_header(header)
       names = Set.new
       header.each_with_index do |name,i|
@@ -173,18 +178,18 @@ module Csvlint
       end
       return valid?
     end
-    
+
     def header?
       @csv_header
     end
-    
+
     def fetch_error(error)
       e = error.message.match(/^(.+?)(?: [io]n)? \(?line \d+\)?\.?$/i)
       message = e[1] rescue nil
       ERROR_MATCHERS.fetch(message, :unknown_error)
     end
-    
-    def dialect_to_csv_options(dialect) 
+
+    def dialect_to_csv_options(dialect)
         skipinitialspace = dialect["skipInitialSpace"] || true
         delimiter = dialect["delimiter"]
         delimiter = delimiter + " " if !skipinitialspace
@@ -195,8 +200,8 @@ module Csvlint
             :skip_blanks => false
         }
     end
-    
-    def build_formats(row) 
+
+    def build_formats(row)
       row.each_with_index do |col, i|
         next if col.nil? || col.empty?
         @formats[i] ||= Hash.new(0)
@@ -228,11 +233,11 @@ module Csvlint
         else
           :string
         end
-        
+
         @formats[i][format] += 1
       end
     end
-    
+
     def check_consistency
       @formats.each_with_index do |format,i|
         if format
@@ -243,10 +248,11 @@ module Csvlint
         end
       end
     end
-    
+
     private
-    
+
     def parse_extension(source)
+      # byebug
       case source
       when File
         return File.extname( source.path )
@@ -254,11 +260,16 @@ module Csvlint
         return ""
       when StringIO
         return ""
-      when Tempfile
+        when Tempfile
+          # this is triggered when the revalidate dialect use case happens
         return ""
       else
-        parsed = URI.parse(source)
-        File.extname(parsed.path)
+        begin
+          parsed = URI.parse(source)
+          File.extname(parsed.path)
+        rescue URI::InvalidURIError
+          return ""
+        end
       end
     end
 

data/lib/csvlint/version.rb

@@ -1,3 +1,3 @@
 module Csvlint
-  VERSION = "0.1.2"
+  VERSION = "0.1.3"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: csvlint
 version: !ruby/object:Gem::Version
-  version: 0.1.2
+  version: 0.1.3
 platform: ruby
 authors:
 - pezholio
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-07-15 00:00:00.000000000 Z
+date: 2015-07-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: mime-types
@@ -248,6 +248,20 @@ dependencies:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: github_changelog_generator
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 description: CSV Validator
 email:
 - pezholio@gmail.com
@@ -261,6 +275,7 @@ files:
 - .gitignore
 - .ruby-version
 - .travis.yml
+- CHANGELOG.md
 - Gemfile
 - LICENSE.md
 - README.md
@@ -270,6 +285,7 @@ files:
 - csvlint.gemspec
 - features/check_format.feature
 - features/csv_options.feature
+- features/csvupload.feature
 - features/fixtures/cr-line-endings.csv
 - features/fixtures/crlf-line-endings.csv
 - features/fixtures/inconsistent-line-endings-unquoted.csv
@@ -336,6 +352,7 @@ summary: CSV Validator
 test_files:
 - features/check_format.feature
 - features/csv_options.feature
+- features/csvupload.feature
 - features/fixtures/cr-line-endings.csv
 - features/fixtures/crlf-line-endings.csv
 - features/fixtures/inconsistent-line-endings-unquoted.csv