csvlint 0.1.4 → 0.2.0

data/spec/csvw/table_group_spec.rb

@@ -0,0 +1,143 @@
+require 'spec_helper'
+
+describe Csvlint::Csvw::TableGroup do
+
+  it "should inherit null to all columns" do
+    @metadata=<<-EOL
+{
+  "@context": "http://www.w3.org/ns/csvw",
+  "null": true,
+  "tables": [{
+    "url": "test040.csv",
+    "tableSchema": {
+      "columns": [{
+        "titles": "null"
+      }, {
+        "titles": "lang"
+      }, {
+        "titles": "textDirection"
+      }, {
+        "titles": "separator"
+      }, {
+        "titles": "ordered"
+      }, {
+        "titles": "default"
+      }, {
+        "titles": "datatype"
+      }, {
+        "titles": "aboutUrl"
+      }, {
+        "titles": "propertyUrl"
+      }, {
+        "titles": "valueUrl"
+      }]
+    }
+  }]
+}
+    EOL
+    json = JSON.parse( @metadata )
+    table_group = Csvlint::Csvw::TableGroup.from_json("http://w3c.github.io/csvw/tests/test040-metadata.json", json)
+
+    expect(table_group.class).to eq(Csvlint::Csvw::TableGroup)
+    expect(table_group.annotations.length).to eq(0)
+    expect(table_group.warnings.length).to eq(1)
+    expect(table_group.warnings[0].type).to eq(:invalid_value)
+    expect(table_group.warnings[0].category).to eq(:metadata)
+    expect(table_group.warnings[0].content).to eq("null: true")
+
+    expect(table_group.tables.length).to eq(1)
+    expect(table_group.tables["http://w3c.github.io/csvw/tests/test040.csv"]).to be_a(Csvlint::Csvw::Table)
+
+    table = table_group.tables["http://w3c.github.io/csvw/tests/test040.csv"]
+    expect(table.columns.length).to eq(10)
+    expect(table.columns[0].null).to eq([""])
+  end
+
+  context "when parsing CSVW table group metadata" do
+
+    before(:each) do
+      @metadata=<<-EOL
+{
+  "@context": "http://www.w3.org/ns/csvw",
+  "tables": [{
+    "url": "countries.csv",
+    "tableSchema": {
+      "columns": [{
+        "name": "countryCode",
+        "titles": "countryCode",
+        "datatype": "string",
+        "propertyUrl": "http://www.geonames.org/ontology{#_name}"
+      }, {
+        "name": "latitude",
+        "titles": "latitude",
+        "datatype": "number"
+      }, {
+        "name": "longitude",
+        "titles": "longitude",
+        "datatype": "number"
+      }, {
+        "name": "name",
+        "titles": "name",
+        "datatype": "string"
+      }],
+      "aboutUrl": "http://example.org/countries.csv{#countryCode}",
+      "propertyUrl": "http://schema.org/{_name}",
+      "primaryKey": "countryCode"
+    }
+  }, {
+    "url": "country_slice.csv",
+    "tableSchema": {
+      "columns": [{
+        "name": "countryRef",
+        "titles": "countryRef",
+        "valueUrl": "http://example.org/countries.csv{#countryRef}"
+      }, {
+        "name": "year",
+        "titles": "year",
+        "datatype": "gYear"
+      }, {
+        "name": "population",
+        "titles": "population",
+        "datatype": "integer"
+      }],
+      "foreignKeys": [{
+        "columnReference": "countryRef",
+        "reference": {
+          "resource": "countries.csv",
+          "columnReference": "countryCode"
+        }
+      }]
+    }
+  }]
+}
+  EOL
+      stub_request(:get, "http://w3c.github.io/csvw/tests/countries.json").to_return(:status => 200, :body => @metadata)
+      @countries=<<-EOL
+countryCode,latitude,longitude,name
+AD,42.546245,1.601554,Andorra
+AE,23.424076,53.847818,"United Arab Emirates"
+AF,33.93911,67.709953,Afghanistan
+  EOL
+      stub_request(:get, "http://w3c.github.io/csvw/tests/countries.csv").to_return(:status => 200, :body => @countries)
+      @country_slice=<<-EOL
+countryRef,year,population
+AF,1960,9616353
+AF,1961,9799379
+AF,1962,9989846
+  EOL
+      stub_request(:get, "http://w3c.github.io/csvw/tests/country_slice.csv").to_return(:status => 200, :body => @country_slice)
+    end
+
+    it "should create a table group from pre-parsed CSVW metadata" do
+      json = JSON.parse( @metadata )
+      table_group = Csvlint::Csvw::TableGroup.from_json("http://w3c.github.io/csvw/tests/countries.json", json)
+
+      expect(table_group.class).to eq(Csvlint::Csvw::TableGroup)
+      expect(table_group.id).to eq(nil)
+      expect(table_group.tables.length).to eq(2)
+      expect(table_group.tables["http://w3c.github.io/csvw/tests/countries.csv"]).to be_a(Csvlint::Csvw::Table)
+      expect(table_group.notes.length).to eq(0)
+      expect(table_group.annotations.length).to eq(0)
+    end
+  end
+end

data/spec/csvw/table_spec.rb

@@ -0,0 +1,90 @@
+require 'spec_helper'
+
+describe Csvlint::Csvw::Table do
+
+  context "when parsing CSVW table metadata" do
+
+    before(:each) do
+      @metadata=<<-EOL
+{
+  "@context": "http://www.w3.org/ns/csvw",
+  "tables": [{
+    "url": "countries.csv",
+    "tableSchema": {
+      "columns": [{
+        "name": "countryCode",
+        "titles": "countryCode",
+        "datatype": "string",
+        "propertyUrl": "http://www.geonames.org/ontology{#_name}"
+      }, {
+        "name": "latitude",
+        "titles": "latitude",
+        "datatype": "number"
+      }, {
+        "name": "longitude",
+        "titles": "longitude",
+        "datatype": "number"
+      }, {
+        "name": "name",
+        "titles": "name",
+        "datatype": "string"
+      }],
+      "aboutUrl": "http://example.org/countries.csv{#countryCode}",
+      "propertyUrl": "http://schema.org/{_name}",
+      "primaryKey": "countryCode"
+    }
+  }, {
+    "url": "country_slice.csv",
+    "tableSchema": {
+      "columns": [{
+        "name": "countryRef",
+        "titles": "countryRef",
+        "valueUrl": "http://example.org/countries.csv{#countryRef}"
+      }, {
+        "name": "year",
+        "titles": "year",
+        "datatype": "gYear"
+      }, {
+        "name": "population",
+        "titles": "population",
+        "datatype": "integer"
+      }],
+      "foreignKeys": [{
+        "columnReference": "countryRef",
+        "reference": {
+          "resource": "countries.csv",
+          "columnReference": "countryCode"
+        }
+      }]
+    }
+  }]
+}
+  EOL
+      stub_request(:get, "http://w3c.github.io/csvw/tests/countries.json").to_return(:status => 200, :body => @metadata)
+      @countries=<<-EOL
+countryCode,latitude,longitude,name
+AD,42.546245,1.601554,Andorra
+AE,23.424076,53.847818,"United Arab Emirates"
+AF,33.93911,67.709953,Afghanistan
+  EOL
+      stub_request(:get, "http://w3c.github.io/csvw/tests/countries.csv").to_return(:status => 200, :body => @countries)
+      @country_slice=<<-EOL
+countryRef,year,population
+AF,1960,9616353
+AF,1961,9799379
+AF,1962,9989846
+  EOL
+      stub_request(:get, "http://w3c.github.io/csvw/tests/country_slice.csv").to_return(:status => 200, :body => @country_slice)
+    end
+
+    it "should create a table from pre-parsed CSVW metadata" do
+      json = JSON.parse( @metadata )
+      table = Csvlint::Csvw::Table.from_json(json["tables"][0], "http://w3c.github.io/csvw/tests/countries.json")
+
+      expect(table).to be_a(Csvlint::Csvw::Table)
+      expect(table.url.to_s).to eq("http://w3c.github.io/csvw/tests/countries.csv")
+      expect(table.columns.length).to eq(4)
+      expect(table.columns[0]).to be_a(Csvlint::Csvw::Column)
+    end
+  end
+end

data/spec/schema_spec.rb

@@ -170,10 +170,11 @@ describe Csvlint::Schema do
       expect( schema.fields[0].constraints["required"] ).to eql(true)
       expect( schema.fields[0].title ).to eql("id")
       expect( schema.fields[0].description ).to eql("house identifier")
+      expect( schema.fields[2].constraints["pattern"]).to eql("[A-Z]{1,2}[0-9][0-9A-Z]? ?[0-9][A-Z]{2}")
     end
 
     it "should create a schema from a JSON Table URL" do
-      schema = Csvlint::Schema.load_from_json_table("http://example.com/example.json")
+      schema = Csvlint::Schema.load_from_json("http://example.com/example.json")
       expect( schema.uri ).to eql("http://example.com/example.json")
       expect( schema.fields.length ).to eql(3)
       expect( schema.fields[0].name ).to eql("ID")
@@ -182,4 +183,29 @@ describe Csvlint::Schema do
     end
   end
 
+  context "when parsing CSVW metadata" do
+
+    before(:each) do
+      @example=<<-EOL
+{
+  "@context": "http://www.w3.org/ns/csvw",
+  "url": "http://example.com/example1.csv",
+  "tableSchema": {
+    "columns": [
+            { "name": "Name", "required": true, "datatype": { "base": "string", "format": ".+" } },
+            { "name": "Id", "required": true, "datatype": { "base": "string", "minLength": 3 } },
+            { "name": "Email", "required": true }
+      ]
+  }
+}
+  EOL
+      stub_request(:get, "http://example.com/metadata.json").to_return(:status => 200, :body => @example)
+    end
+
+    it "should create a table group from a CSVW metadata URL" do
+      schema = Csvlint::Schema.load_from_json("http://example.com/metadata.json")
+      expect( schema.class ).to eq(Csvlint::Csvw::TableGroup)
+    end
+  end
+
 end

data/spec/spec_helper.rb

@@ -6,7 +6,6 @@ require 'pry'
 require 'webmock/rspec'
 
 RSpec.configure do |config|
-  config.treat_symbols_as_metadata_keys_with_true_values = true
   config.run_all_when_everything_filtered = true
   config.filter_run :focus
 

data/spec/validator_spec.rb

@@ -4,13 +4,16 @@ describe Csvlint::Validator do
     
   before do
     stub_request(:get, "http://example.com/example.csv").to_return(:status => 200, :body => "")
+    stub_request(:get, "http://example.com/.well-known/csvm").to_return(:status => 404)
+    stub_request(:get, "http://example.com/example.csv-metadata.json").to_return(:status => 404)
+    stub_request(:get, "http://example.com/csv-metadata.json").to_return(:status => 404)
   end
   
   context "csv dialect" do
     it "should provide sensible defaults for CSV parsing" do
       validator = Csvlint::Validator.new("http://example.com/example.csv")
       opts = validator.instance_variable_get("@csv_options")
-      opts.should include({
+      expect(opts).to include({
         :col_sep => ",",
         :row_sep => :auto,
         :quote_char => '"',
@@ -25,7 +28,7 @@ describe Csvlint::Validator do
         "delimiter" => "\t",
         "quoteChar" => "'"
       })
-      opts.should include({
+      expect(opts).to include({
         :col_sep => "\t",
         :row_sep => "\n",
         :quote_char => "'",
@@ -93,6 +96,7 @@ describe Csvlint::Validator do
     it "should warn if column names aren't unique" do      
       data = StringIO.new( "minimum, minimum" )
       validator = Csvlint::Validator.new(data)
+      validator.reset
       expect( validator.validate_header(["minimum", "minimum"]) ).to eql(true)
       expect( validator.warnings.size ).to eql(1)
       expect( validator.warnings.first.type).to eql(:duplicate_column_name)
@@ -173,7 +177,7 @@ describe Csvlint::Validator do
         validator.build_formats(row)
         formats = validator.instance_variable_get("@formats")
         
-        formats[0].keys.first.should == type
+        expect(formats[0].keys.first).to eql type
       end
     end
     
@@ -184,8 +188,8 @@ describe Csvlint::Validator do
       validator.build_formats(row)
       formats = validator.instance_variable_get("@formats")
       
-      formats[0].keys.first.should == :numeric
-      formats[1].keys.first.should == :numeric
+      expect(formats[0].keys.first).to eql :numeric
+      expect(formats[1].keys.first).to eql :numeric
     end
   
     it "should ignore blank arrays" do
@@ -194,7 +198,7 @@ describe Csvlint::Validator do
       validator = Csvlint::Validator.new("http://example.com/example.csv")
       validator.build_formats(row)
       formats = validator.instance_variable_get("@formats") 
-      formats.should == []
+      expect(formats).to eql []
     end
     
     it "should work correctly for single columns" do
@@ -212,7 +216,7 @@ describe Csvlint::Validator do
       
       formats = validator.instance_variable_get("@formats")
       
-      formats.should == [{:string => 3}]
+      expect(formats).to eql [{:string => 3}]
     end
   
     it "should return formats correctly if a row is blank" do
@@ -229,7 +233,7 @@ describe Csvlint::Validator do
       
       formats = validator.instance_variable_get("@formats") 
             
-      formats.should == [
+      expect(formats).to eql [
         {:string => 1},
         {:numeric => 1},
         {:string => 1},
@@ -254,7 +258,7 @@ describe Csvlint::Validator do
       warnings = validator.instance_variable_get("@warnings") 
       warnings.delete_if { |h| h.type != :inconsistent_values }
       
-      warnings.count.should == 1
+      expect(warnings.count).to eql 1
     end
     
   end
@@ -263,12 +267,13 @@ describe Csvlint::Validator do
    
     before :all do
       stub_request(:get, "http://example.com/crlf.csv").to_return(:status => 200, :body => File.read(File.join(File.dirname(__FILE__),'..','features','fixtures','windows-line-endings.csv')))
+      stub_request(:get, "http://example.com/crlf.csv-metadata.json").to_return(:status => 404)
     end
     
     it "can get line break symbol" do
 
       validator = Csvlint::Validator.new("http://example.com/crlf.csv")
-      validator.line_breaks.should == "\r\n"
+      expect(validator.line_breaks).to eql "\r\n"
       
     end
     
@@ -299,6 +304,7 @@ describe Csvlint::Validator do
   
   it "should follow redirects to SSL" do
     stub_request(:get, "http://example.com/redirect").to_return(:status => 301, :headers=>{"Location" => "https://example.com/example.csv"})
+    stub_request(:get, "http://example.com/redirect-metadata.json").to_return(:status => 404)
     stub_request(:get, "https://example.com/example.csv").to_return(:status => 200, 
         :headers=>{"Content-Type" => "text/csv; header=present"}, 
         :body => File.read(File.join(File.dirname(__FILE__),'..','features','fixtures','valid.csv')))

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: csvlint
 version: !ruby/object:Gem::Version
-  version: 0.1.4
+  version: 0.2.0
 platform: ruby
 authors:
 - pezholio
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-08-06 00:00:00.000000000 Z
+date: 2015-10-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: mime-types
@@ -80,6 +80,34 @@ dependencies:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: escape_utils
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: uri_template
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -286,6 +314,7 @@ files:
 - features/check_format.feature
 - features/csv_options.feature
 - features/csvupload.feature
+- features/csvw_schema_validation.feature
 - features/fixtures/cr-line-endings.csv
 - features/fixtures/crlf-line-endings.csv
 - features/fixtures/inconsistent-line-endings-unquoted.csv
@@ -293,6 +322,7 @@ files:
 - features/fixtures/invalid-byte-sequence.csv
 - features/fixtures/lf-line-endings.csv
 - features/fixtures/spreadsheet.xls
+- features/fixtures/spreadsheet.xlsx
 - features/fixtures/title-row.csv
 - features/fixtures/valid.csv
 - features/fixtures/windows-line-endings.csv
@@ -309,11 +339,19 @@ files:
 - features/step_definitions/validation_info_steps.rb
 - features/step_definitions/validation_warnings_steps.rb
 - features/support/env.rb
+- features/support/load_tests.rb
 - features/support/webmock.rb
 - features/validation_errors.feature
 - features/validation_info.feature
 - features/validation_warnings.feature
 - lib/csvlint.rb
+- lib/csvlint/csvw/column.rb
+- lib/csvlint/csvw/date_format.rb
+- lib/csvlint/csvw/metadata_error.rb
+- lib/csvlint/csvw/number_format.rb
+- lib/csvlint/csvw/property_checker.rb
+- lib/csvlint/csvw/table.rb
+- lib/csvlint/csvw/table_group.rb
 - lib/csvlint/error_collector.rb
 - lib/csvlint/error_message.rb
 - lib/csvlint/field.rb
@@ -321,6 +359,11 @@ files:
 - lib/csvlint/validate.rb
 - lib/csvlint/version.rb
 - lib/csvlint/wrapped_io.rb
+- spec/csvw/column_spec.rb
+- spec/csvw/date_format_spec.rb
+- spec/csvw/number_format_spec.rb
+- spec/csvw/table_group_spec.rb
+- spec/csvw/table_spec.rb
 - spec/field_spec.rb
 - spec/schema_spec.rb
 - spec/spec_helper.rb
@@ -353,6 +396,7 @@ test_files:
 - features/check_format.feature
 - features/csv_options.feature
 - features/csvupload.feature
+- features/csvw_schema_validation.feature
 - features/fixtures/cr-line-endings.csv
 - features/fixtures/crlf-line-endings.csv
 - features/fixtures/inconsistent-line-endings-unquoted.csv
@@ -360,6 +404,7 @@ test_files:
 - features/fixtures/invalid-byte-sequence.csv
 - features/fixtures/lf-line-endings.csv
 - features/fixtures/spreadsheet.xls
+- features/fixtures/spreadsheet.xlsx
 - features/fixtures/title-row.csv
 - features/fixtures/valid.csv
 - features/fixtures/windows-line-endings.csv
@@ -376,10 +421,16 @@ test_files:
 - features/step_definitions/validation_info_steps.rb
 - features/step_definitions/validation_warnings_steps.rb
 - features/support/env.rb
+- features/support/load_tests.rb
 - features/support/webmock.rb
 - features/validation_errors.feature
 - features/validation_info.feature
 - features/validation_warnings.feature
+- spec/csvw/column_spec.rb
+- spec/csvw/date_format_spec.rb
+- spec/csvw/number_format_spec.rb
+- spec/csvw/table_group_spec.rb
+- spec/csvw/table_spec.rb
 - spec/field_spec.rb
 - spec/schema_spec.rb
 - spec/spec_helper.rb