csvlint 0.2.6 → 0.3.0

data/lib/csvlint/csvw/table_group.rb

@@ -20,22 +20,22 @@ module Csvlint
         @warnings += @tables.map{|url,table| table.warnings}.flatten
       end
 
-      def validate_header(header, table_url)
+      def validate_header(header, table_url, strict)
         reset
         table_url = "file:#{File.absolute_path(table_url)}" if table_url.instance_of? File
         table = tables[table_url]
-        table.validate_header(header)
+        table.validate_header(header, strict)
         @errors += table.errors
         @warnings += table.warnings
         return valid?
       end
 
-      def validate_row(values, row=nil, all_errors=[], table_url)
+      def validate_row(values, row=nil, all_errors=[], table_url, validate)
         reset
         table_url = "file:#{File.absolute_path(table_url)}" if table_url.instance_of? File
         @validated_tables[table_url] = true
         table = tables[table_url]
-        table.validate_row(values, row)
+        table.validate_row(values, row, validate)
         @errors += table.errors
         @warnings += table.warnings
         return valid?
@@ -93,10 +93,10 @@ module Csvlint
               annotations[property] = v
             elsif type == :common
               common_properties[property] = v
-            elsif type == :column
-              warnings << Csvlint::ErrorMessage.new(:invalid_property, :metadata, nil, nil, "#{property}", nil)
-            else
+            elsif type == :inherited
               inherited_properties[property] = v
+            else
+              warnings << Csvlint::ErrorMessage.new(:invalid_property, :metadata, nil, nil, "#{property}", nil)
             end
           end
         end
@@ -118,7 +118,7 @@ module Csvlint
             end
             table_url = URI.join(base_url, table_url).to_s
             table_desc["url"] = table_url
-            table = Csvlint::Csvw::Table.from_json(table_desc, base_url, lang, inherited_properties)
+            table = Csvlint::Csvw::Table.from_json(table_desc, base_url, lang, common_properties, inherited_properties)
             tables[table_url] = table
           else
             warnings << Csvlint::ErrorMessage.new(:invalid_table_description, :metadata, nil, nil, "#{table_desc}", nil)
@@ -154,7 +154,7 @@ module Csvlint
           end
         end
 
-        return self.new(base_url, id: id, tables: tables, notes: json["notes"] || [], annotations: annotations, warnings: warnings)
+        return self.new(base_url, id: id, tables: tables, notes: common_properties["notes"] || [], annotations: annotations, warnings: warnings)
       end
 
       private

data/lib/csvlint/schema.rb

@@ -54,7 +54,7 @@ module Csvlint
 
     end
 
-    def validate_header(header, source_url=nil)
+    def validate_header(header, source_url=nil, validate=true)
       reset
 
       found_header = header.to_csv(:row_sep => '')
@@ -65,7 +65,7 @@ module Csvlint
       return valid?
     end
 
-    def validate_row(values, row=nil, all_errors=[], source_url=nil)
+    def validate_row(values, row=nil, all_errors=[], source_url=nil, validate=true)
       reset
       if values.length < fields.length
         fields[values.size..-1].each_with_index do |field, i|

data/lib/csvlint/validate.rb

@@ -68,6 +68,7 @@ module Csvlint
       @csv_header = true
       @headers = {}
       @lambda = options[:lambda]
+      @validate = options[:validate].nil? ? true : options[:validate]
       @leading = ""
 
       @limit_lines = options[:limit_lines]
@@ -151,6 +152,7 @@ module Csvlint
       end
     rescue ArgumentError => ae
       build_errors(:invalid_encoding, :structure, @current_line, nil, @current_line) unless @reported_invalid_encoding
+      @current_line = @current_line+1
       @reported_invalid_encoding = true
     end
 
@@ -181,7 +183,6 @@ module Csvlint
         build_exception_messages(e, stream, current_line)
       end
 
-      @data << row
       if row
         if current_line <= 1 && @csv_header
           # this conditional should be refactored somewhere
@@ -195,7 +196,7 @@ module Csvlint
           build_errors(:blank_rows, :structure, current_line, nil, stream.to_s) if row.reject { |c| c.nil? || c.empty? }.size == 0
           # Builds errors and warnings related to the provided schema file
           if @schema
-            @schema.validate_row(row, current_line, all_errors, @source)
+            @schema.validate_row(row, current_line, all_errors, @source, @validate)
             @errors += @schema.errors
             all_errors += @schema.errors
             @warnings += @schema.warnings
@@ -204,6 +205,7 @@ module Csvlint
           end
         end
       end
+      @data << row
     end
 
     def finish
@@ -214,7 +216,7 @@ module Csvlint
       # return expected_columns to calling class
       build_warnings(:check_options, :structure) if @expected_columns == 1
       check_consistency
-      check_foreign_keys
+      check_foreign_keys if @validate
       check_mixed_linebreaks
       validate_encoding
     end
@@ -250,7 +252,7 @@ module Csvlint
             schema = Schema.load_from_json(url)
             if schema.instance_of? Csvlint::Csvw::TableGroup
               if schema.tables[@source_url]
-                link_schema = schema
+                @schema = schema
               else
                 warn_if_unsuccessful = true
                 build_warnings(:schema_mismatch, :context, nil, nil, @source_url, schema)
@@ -293,6 +295,7 @@ module Csvlint
       end
       @dialect = {
           "header" => true,
+          "headerRowCount" => 1,
           "delimiter" => ",",
           "skipInitialSpace" => true,
           "lineTerminator" => :auto,
@@ -358,7 +361,7 @@ module Csvlint
         end
       end
       if @schema
-        @schema.validate_header(header, @source)
+        @schema.validate_header(header, @source, @validate)
         @errors += @schema.errors
         @warnings += @schema.warnings
       end
@@ -441,15 +444,11 @@ module Csvlint
           @schema = nil
         end
       end
-      link_schema = nil
-      @schema = link_schema if link_schema
-
       paths = []
       if @source_url =~ /^http(s)?/
         begin
           well_known_uri = URI.join(@source_url, "/.well-known/csvm")
-          well_known = open(well_known_uri).read
-            # TODO
+          paths = open(well_known_uri).read.split("\n")
         rescue OpenURI::HTTPError, URI::BadURIError
         end
       end
@@ -464,6 +463,7 @@ module Csvlint
           if schema.instance_of? Csvlint::Csvw::TableGroup
             if schema.tables[@source_url]
               @schema = schema
+              return
             else
               warn_if_unsuccessful = true
               build_warnings(:schema_mismatch, :context, nil, nil, @source_url, schema)
@@ -472,9 +472,6 @@ module Csvlint
         rescue Errno::ENOENT
         rescue OpenURI::HTTPError, URI::BadURIError, ArgumentError
         rescue => e
-          STDERR.puts e.class
-          STDERR.puts e.message
-          STDERR.puts e.backtrace
           raise e
         end
       end

data/lib/csvlint/version.rb

@@ -1,3 +1,3 @@
 module Csvlint
-  VERSION = "0.2.6"
+  VERSION = "0.3.0"
 end

data/spec/csvw/column_spec.rb

@@ -4,21 +4,21 @@ describe Csvlint::Csvw::Column do
 
   it "shouldn't generate errors for string values" do
     column = Csvlint::Csvw::Column.new(1, "foo")
-    valid = column.validate("bar", 2)
-    expect(valid).to eq(true)
+    value = column.validate("bar", 2)
+    expect(value).to eq("bar")
   end
 
   it "should generate errors for string values that aren't long enough" do
     column = Csvlint::Csvw::Column.new(1, "foo", datatype: { "base" => "http://www.w3.org/2001/XMLSchema#string", "minLength" => 4 })
-    valid = column.validate("bar", 2)
-    expect(valid).to eq(false)
+    value = column.validate("bar", 2)
+    expect(value).to eq({ :invalid => "bar" })
     expect(column.errors.length).to eq(1)
   end
 
   it "shouldn't generate errors for string values that are long enough" do
     column = Csvlint::Csvw::Column.new(1, "foo", datatype: { "base" => "http://www.w3.org/2001/XMLSchema#string", "minLength" => 4 })
-    valid = column.validate("barn", 2)
-    expect(valid).to eq(true)
+    value = column.validate("barn", 2)
+    expect(value).to eq("barn")
     expect(column.errors.length).to eq(0)
   end
 

data/spec/csvw/date_format_spec.rb

@@ -1,40 +1,40 @@
 require 'spec_helper'
 
-describe Csvlint::Csvw::NumberFormat do
+describe Csvlint::Csvw::DateFormat do
 
   it "should parse dates that match yyyy-MM-dd correctly" do
     format = Csvlint::Csvw::DateFormat.new("yyyy-MM-dd")
-    expect(format.parse("2015-03-22")).to eql(Date.new(2015, 3, 22))
+    expect(format.parse("2015-03-22")[:dateTime]).to eql(Date.new(2015, 3, 22))
     expect(format.parse("2015-02-30")).to eql(nil)
     expect(format.parse("22/03/2015")).to eq(nil)
   end
 
   it "should parse times that match HH:mm:ss correctly" do
     format = Csvlint::Csvw::DateFormat.new("HH:mm:ss")
-    expect(format.parse("12:34:56")).to eql({ "hour" => 12, "minute" => 34, "second" => 56.0 })
+    expect(format.parse("12:34:56")).to eql({ :hour => 12, :minute => 34, :second => 56.0, :string => "12:34:56", :dateTime => DateTime.new(0,1,1,12,34,56.0,"+00:00") })
     expect(format.parse("22/03/2015")).to eq(nil)
   end
 
   it "should parse times that match HH:mm:ss.SSS correctly" do
     format = Csvlint::Csvw::DateFormat.new("HH:mm:ss.SSS")
-    expect(format.parse("12:34:56")).to eql({ "hour" => 12, "minute" => 34, "second" => 56.0 })
-    expect(format.parse("12:34:56.78")).to eql({ "hour" => 12, "minute" => 34, "second" => 56.78 })
-    expect(format.parse("12:34:56.789")).to eql({ "hour" => 12, "minute" => 34, "second" => 56.789 })
+    expect(format.parse("12:34:56")).to eql({ :hour => 12, :minute => 34, :second => 56.0, :string => "12:34:56", :dateTime => DateTime.new(0,1,1,12,34,56.0,"+00:00") })
+    expect(format.parse("12:34:56.78")).to eql({ :hour => 12, :minute => 34, :second => 56.78, :string => "12:34:56.78", :dateTime => DateTime.new(0,1,1,12,34,56.78,"+00:00") })
+    expect(format.parse("12:34:56.789")).to eql({ :hour => 12, :minute => 34, :second => 56.789, :string => "12:34:56.789", :dateTime => DateTime.new(0,1,1,12,34,56.789,"+00:00") })
     expect(format.parse("12:34:56.7890")).to eql(nil)
     expect(format.parse("22/03/2015")).to eq(nil)
   end
 
   it "should parse dateTimes that match yyyy-MM-ddTHH:mm:ss correctly" do
     format = Csvlint::Csvw::DateFormat.new("yyyy-MM-ddTHH:mm:ss")
-    expect(format.parse("2015-03-15T15:02:37")).to eql(DateTime.new(2015, 3, 15, 15, 2, 37))
+    expect(format.parse("2015-03-15T15:02:37")[:dateTime]).to eql(DateTime.new(2015, 3, 15, 15, 2, 37))
     expect(format.parse("12:34:56")).to eql(nil)
     expect(format.parse("22/03/2015")).to eq(nil)
   end
 
   it "should parse dateTimes that match yyyy-MM-ddTHH:mm:ss.S correctly" do
     format = Csvlint::Csvw::DateFormat.new("yyyy-MM-ddTHH:mm:ss.S")
-    expect(format.parse("2015-03-15T15:02:37")).to eql(DateTime.new(2015, 3, 15, 15, 2, 37.0))
-    expect(format.parse("2015-03-15T15:02:37.4")).to eql(DateTime.new(2015, 3, 15, 15, 2, 37.4))
+    expect(format.parse("2015-03-15T15:02:37")[:dateTime]).to eql(DateTime.new(2015, 3, 15, 15, 2, 37.0))
+    expect(format.parse("2015-03-15T15:02:37.4")[:dateTime]).to eql(DateTime.new(2015, 3, 15, 15, 2, 37.4))
     expect(format.parse("2015-03-15T15:02:37.45")).to eql(nil)
     expect(format.parse("12:34:56")).to eql(nil)
     expect(format.parse("22/03/2015")).to eq(nil)
@@ -43,7 +43,7 @@ describe Csvlint::Csvw::NumberFormat do
   it "should parse dateTimes that match M/d/yyyy HH:mm correctly" do
     format = Csvlint::Csvw::DateFormat.new("M/d/yyyy HH:mm")
     expect(format.parse("2015-03-15T15:02:37")).to eql(nil)
-    expect(format.parse("3/15/2015 15:02")).to eql(DateTime.new(2015, 3, 15, 15, 2))
+    expect(format.parse("3/15/2015 15:02")[:dateTime]).to eql(DateTime.new(2015, 3, 15, 15, 2))
   end
 
 end

data/spec/csvw/number_format_spec.rb

@@ -152,6 +152,7 @@ describe Csvlint::Csvw::NumberFormat do
 
   it "should parse numbers that match ##0 correctly" do
     format = Csvlint::Csvw::NumberFormat.new("##0")
+    expect(format.parse("-1")).to eql(-1)
     expect(format.parse("1")).to eql(1)
     expect(format.parse("12")).to eql(12)
     expect(format.parse("123")).to eql(123)
@@ -380,6 +381,21 @@ describe Csvlint::Csvw::NumberFormat do
     expect(format.parse("12.34E5")).to eql(12.34E5)
   end
 
+  it "should parse numbers that match %000 correctly" do
+    format = Csvlint::Csvw::NumberFormat.new("%000")
+    expect(format.parse("%001")).to eq(0.01)
+    expect(format.parse("%012")).to eq(0.12)
+    expect(format.parse("%123")).to eq(1.23)
+    expect(format.parse("%1234")).to eq(12.34)
+  end
+
+  it "should parse numbers that match -0 correctly" do
+    format = Csvlint::Csvw::NumberFormat.new("-0")
+    expect(format.parse("1")).to eq(nil)
+    expect(format.parse("-1")).to eq(-1)
+    expect(format.parse("-12")).to eq(-12)
+  end
+
   it "should parse numbers normally when there is no pattern" do
     format = Csvlint::Csvw::NumberFormat.new()
     expect(format.parse("1")).to eql(1)
@@ -414,4 +430,17 @@ describe Csvlint::Csvw::NumberFormat do
     expect(format.parse("-INF")).to eql(-Float::INFINITY)
   end
 
+  it "should parse numbers including decimal separators when they are specified" do
+    format = Csvlint::Csvw::NumberFormat.new(nil, " ", ",")
+    expect(format.parse("1")).to eql(1)
+    expect(format.parse("12,3")).to eql(12.3)
+    expect(format.parse("12,34")).to eql(12.34)
+    expect(format.parse("12,3E4")).to eql(12.3E4)
+    expect(format.parse("12,3E45")).to eql(12.3E45)
+    expect(format.parse("12,34E5")).to eql(12.34E5)
+    expect(format.parse("1 234")).to eql(1234)
+    expect(format.parse("1 234 567")).to eql(1234567)
+    expect(format.parse("1  234")).to eq(nil)
+  end
+
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: csvlint
 version: !ruby/object:Gem::Version
-  version: 0.2.6
+  version: 0.3.0
 platform: ruby
 authors:
 - pezholio
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-11-16 00:00:00.000000000 Z
+date: 2016-01-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: mime-types
@@ -332,6 +332,34 @@ dependencies:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: rdf
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rdf-turtle
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 description: CSV Validator
 email:
 - pezholio@gmail.com
@@ -371,6 +399,7 @@ files:
 - features/fixtures/title-row.csv
 - features/fixtures/valid.csv
 - features/fixtures/valid_many_rows.csv
+- features/fixtures/w3.org/.well-known/csvm
 - features/fixtures/windows-line-endings.csv
 - features/information.feature
 - features/parse_csv.feature
@@ -386,6 +415,7 @@ files:
 - features/step_definitions/validation_info_steps.rb
 - features/step_definitions/validation_warnings_steps.rb
 - features/support/aruba.rb
+- features/support/earl_formatter.rb
 - features/support/env.rb
 - features/support/load_tests.rb
 - features/support/webmock.rb
@@ -458,6 +488,7 @@ test_files:
 - features/fixtures/title-row.csv
 - features/fixtures/valid.csv
 - features/fixtures/valid_many_rows.csv
+- features/fixtures/w3.org/.well-known/csvm
 - features/fixtures/windows-line-endings.csv
 - features/information.feature
 - features/parse_csv.feature
@@ -473,6 +504,7 @@ test_files:
 - features/step_definitions/validation_info_steps.rb
 - features/step_definitions/validation_warnings_steps.rb
 - features/support/aruba.rb
+- features/support/earl_formatter.rb
 - features/support/env.rb
 - features/support/load_tests.rb
 - features/support/webmock.rb