csvlint 0.1.4 → 0.2.0

data/features/sources.feature

@@ -1,5 +1,5 @@
 Feature: Parse CSV from Different Sources
-  
+
   Scenario: Successfully parse a valid CSV from a StringIO
       Given I have a CSV with the following content:
     """
@@ -10,9 +10,8 @@ Feature: Parse CSV from Different Sources
     And it is parsed as a StringIO
     When I ask if the CSV is valid
     Then I should get the value of true
-  
+
   Scenario: Successfully parse a valid CSV from a File
-    Given I parse a CSV file called "valid.csv"
+    Given I parse a file called "valid.csv"
     When I ask if the CSV is valid
     Then I should get the value of true
-  

data/features/step_definitions/parse_csv_steps.rb

@@ -2,17 +2,29 @@ Given(/^I have a CSV with the following content:$/) do |string|
   @csv = string.to_s
 end
 
+Given(/^it has a Link header holding "(.*?)"$/) do |link|
+  @link = "#{link}; type=\"application/csvm+json\""
+end
+
 Given(/^it is stored at the url "(.*?)"$/) do |url|
   @url = url
   content_type = @content_type || "text/csv"
   charset = @encoding || "UTF-8"
-  stub_request(:get, url).to_return(:status => 200, :body => @csv, :headers => {"Content-Type" => "#{content_type}; charset=#{charset}"})
+  headers = {"Content-Type" => "#{content_type}; charset=#{charset}"}
+  headers["Link"] = @link if @link
+  stub_request(:get, url).to_return(:status => 200, :body => @csv, :headers => headers)
+  stub_request(:get, URI.join(url, '/.well-known/csvm')).to_return(:status => 404)
+  stub_request(:get, url + '-metadata.json').to_return(:status => 404)
+  stub_request(:get, URI.join(url, 'csv-metadata.json')).to_return(:status => 404)
 end
 
 Given(/^it is stored at the url "(.*?)" with no character set$/) do |url|
   @url = url
   content_type = @content_type || "text/csv"
   stub_request(:get, url).to_return(:status => 200, :body => @csv, :headers => {"Content-Type" => "#{content_type}"})
+  stub_request(:get, URI.join(url, '/.well-known/csvm')).to_return(:status => 404)
+  stub_request(:get, url + '-metadata.json').to_return(:status => 404)
+  stub_request(:get, URI.join(url, 'csv-metadata.json')).to_return(:status => 404)
 end
 
 When(/^I ask if the CSV is valid$/) do

data/features/step_definitions/schema_validation_steps.rb

@@ -1,7 +1,33 @@
 Given(/^I have a schema with the following content:$/) do |json|
+  @schema_type = :json_table
   @schema_json = json
 end
 
-Given(/^the schema is stored at the url "(.*?)"$/) do |schema_url|
+Given(/^I have metadata with the following content:$/) do |json|
+  @schema_type = :csvw_metadata
+  @schema_json = json
+end
+
+Given(/^I have a metadata file called "([^"]*)"$/) do |filename|
+  @schema_type = :csvw_metadata
+  @schema_json = File.read( File.join( File.dirname(__FILE__), "..", "fixtures", filename ) )
+end
+
+Given(/^the (schema|metadata) is stored at the url "(.*?)"$/) do |schema_type,schema_url|
   @schema_url = schema_url
+  stub_request(:get, @schema_url).to_return(:status => 200, :body => @schema_json.to_str)
+end
+
+Given(/^there is a file at "(.*?)" with the content:$/) do |url, content|
+  stub_request(:get, url).to_return(:status => 200, :body => content.to_str)
+end
+
+Given(/^I have a file called "(.*?)" at the url "(.*?)"$/) do |filename,url|
+  content = File.read( File.join( File.dirname(__FILE__), "..", "fixtures", filename ) )
+  content_type = filename =~ /.csv$/ ? "text/csv" : "application/csvm+json"
+  stub_request(:get, url).to_return(:status => 200, :body => content, :headers => {"Content-Type" => "#{content_type}; charset=UTF-8"})
+end
+
+Given(/^there is no file at the url "(.*?)"$/) do |url|
+  stub_request(:get, url).to_return(:status => 404)
 end

data/features/step_definitions/sources_steps.rb

@@ -2,6 +2,6 @@ Given(/^it is parsed as a StringIO$/) do
   @url = StringIO.new(@csv)
 end
 
-Given(/^I parse a CSV file called "(.*?)"$/) do |filename|
+Given(/^I parse a file called "(.*?)"$/) do |filename|
   @url = File.new( File.join( File.dirname(__FILE__), "..", "fixtures", filename ) )
 end

data/features/step_definitions/validation_errors_steps.rb

@@ -2,13 +2,60 @@ When(/^I ask if there are errors$/) do
   @csv_options ||= default_csv_options
 
   if @schema_json
-    @schema = Csvlint::Schema.from_json_table( @schema_url || "http://example.org ", JSON.parse(@schema_json) )
+    if @schema_type == :json_table
+      @schema = Csvlint::Schema.from_json_table( @schema_url || "http://example.org ", JSON.parse(@schema_json) )
+    else
+      @schema = Csvlint::Schema.from_csvw_metadata( @schema_url || "http://example.org ", JSON.parse(@schema_json) )
+    end
   end
 
   @validator = Csvlint::Validator.new( @url, @csv_options, @schema )
   @errors = @validator.errors
 end
 
+When(/^I carry out CSVW validation$/) do
+  @csv_options ||= default_csv_options
+
+  begin
+    if @schema_json
+      json = JSON.parse(@schema_json)
+      if @schema_type == :json_table
+        @schema = Csvlint::Schema.from_json_table( @schema_url || "http://example.org ", json )
+      else
+        @schema = Csvlint::Schema.from_csvw_metadata( @schema_url || "http://example.org ", json )
+      end
+    end
+
+    if @url.nil?
+      @errors = []
+      @warnings = []
+      @schema.tables.keys.each do |table_url|
+        validator = Csvlint::Validator.new( table_url, @csv_options, @schema )
+        @errors += validator.errors
+        @warnings += validator.warnings
+      end
+    else
+      validator = Csvlint::Validator.new( @url, @csv_options, @schema )
+      @errors = validator.errors
+      @warnings = validator.warnings
+    end
+  rescue JSON::ParserError => e
+    @errors = [e]
+  rescue Csvlint::Csvw::MetadataError => e
+    @errors = [e]
+  end
+end
+
+Then(/^there should be errors$/) do
+  # this test is only used for CSVW testing; :invalid_encoding masks lack of real errors
+  @errors.delete_if { |e| e.instance_of?(Csvlint::ErrorMessage) && [:invalid_encoding, :line_breaks].include?(e.type) }
+  expect( @errors.count ).to be > 0
+end
+
+Then(/^there should not be errors$/) do
+  expect( @errors.count ).to eq(0)
+end
+
 Then(/^there should be (\d+) error$/) do |count|
   expect( @errors.count ).to eq( count.to_i )
 end

data/features/step_definitions/validation_info_steps.rb

@@ -2,7 +2,11 @@ Given(/^I ask if there are info messages$/) do
   @csv_options ||= default_csv_options
   
   if @schema_json
-    @schema = Csvlint::Schema.from_json_table( @schema_url || "http://example.org ", JSON.parse(@schema_json) )
+    if @schema_type == :json_table
+      @schema = Csvlint::Schema.from_json_table( @schema_url || "http://example.org ", JSON.parse(@schema_json) )
+    else
+      @schema = Csvlint::Schema.from_csvw_metadata( @schema_url || "http://example.org ", JSON.parse(@schema_json) )
+    end
   end
   
   @validator = Csvlint::Validator.new( @url, @csv_options, @schema ) 

data/features/step_definitions/validation_warnings_steps.rb

@@ -18,13 +18,27 @@ end
 When(/^I ask if there are warnings$/) do
   @csv_options ||= default_csv_options
   if @schema_json
-    @schema = Csvlint::Schema.from_json_table( @schema_url || "http://example.org ", JSON.parse(@schema_json) )
+    if @schema_type == :json_table
+      @schema = Csvlint::Schema.from_json_table( @schema_url || "http://example.org ", JSON.parse(@schema_json) )
+    else
+      @schema = Csvlint::Schema.from_csvw_metadata( @schema_url || "http://example.org ", JSON.parse(@schema_json) )
+    end
   end
 
   @validator = Csvlint::Validator.new( @url, @csv_options, @schema )
   @warnings = @validator.warnings
 end
 
+Then(/^there should be warnings$/) do
+  expect( @warnings.count ).to be > 0
+end
+
+Then(/^there should not be warnings$/) do
+  # this test is only used for CSVW testing, and :inconsistent_values warnings don't count in CSVW
+  @warnings.delete_if { |w| [:inconsistent_values, :check_options].include?(w.type) }
+  expect( @warnings.count ).to eq(0)
+end
+
 Then(/^there should be (\d+) warnings$/) do |count|
   expect( @warnings.count ).to eq( count.to_i )
 end

data/features/support/load_tests.rb

@@ -0,0 +1,114 @@
+require 'json'
+require 'open-uri'
+require 'uri'
+
+BASE_URI = "http://w3c.github.io/csvw/tests/"
+BASE_PATH = File.join(File.dirname(__FILE__), "..", "fixtures", "csvw")
+FEATURE_FILE_PATH = File.join(File.dirname(__FILE__), "..", "csvw_validation_tests.feature")
+SCRIPT_FILE_PATH = File.join(File.dirname(__FILE__), "..", "..", "bin", "run-csvw-tests")
+
+Dir.mkdir(BASE_PATH) unless Dir.exist?(BASE_PATH)
+
+def cache_file(filename)
+	file = File.join(BASE_PATH, filename)
+	uri = URI.join(BASE_URI, filename)
+	unless File.exist?(file)
+		if filename.include? "/"
+			levels = filename.split("/")[0..-2]
+			for i in 0..levels.length
+				dir = File.join(BASE_PATH, levels[0..i].join("/"))
+				Dir.mkdir(dir) unless Dir.exist?(dir)
+			end
+		end
+		STDERR.puts("storing #{file} locally")
+		File.open(file, 'wb') do |f|
+			f.puts open(uri, 'rb').read
+		end
+	end
+	return uri, file
+end
+
+File.open(SCRIPT_FILE_PATH, 'w') do |file|
+	File.chmod(0755, SCRIPT_FILE_PATH)
+	manifest = JSON.parse( open("http://w3c.github.io/csvw/tests/manifest-validation.jsonld").read )
+	manifest["entries"].each do |entry|
+		type = "valid"
+		case entry["type"] 
+		when "csvt:WarningValidationTest" 
+			type = "warnings" 
+		when "csvt:NegativeValidationTest" 
+			type = "errors" 
+		end
+		file.puts "echo \"#{entry["id"].split("#")[-1]}: #{entry["name"].gsub("`", "'")}\""
+		file.puts "echo \"#{type}: #{entry["comment"].gsub("\"", "\\\"").gsub("`", "'")}\""
+		if entry["action"].end_with?(".json")
+			file.puts "csvlint --schema=features/fixtures/csvw/#{entry["action"]}"
+		elsif entry["option"] && entry["option"]["metadata"]
+			file.puts "csvlint features/fixtures/csvw/#{entry["action"]} --schema=features/fixtures/csvw/#{entry["option"]["metadata"]}"
+		else
+			file.puts "csvlint features/fixtures/csvw/#{entry["action"]}"
+		end
+		file.puts "echo"
+	end
+end unless File.exist? SCRIPT_FILE_PATH
+
+File.open(FEATURE_FILE_PATH, 'w') do |file|
+	file.puts "# Auto-generated file based on standard validation CSVW tests from http://w3c.github.io/csvw/tests/manifest-validation.jsonld"
+	file.puts ""
+
+	manifest = JSON.parse( open("http://w3c.github.io/csvw/tests/manifest-validation.jsonld").read )
+
+	file.puts "Feature: #{manifest["label"]}"
+	file.puts ""
+	
+	manifest["entries"].each do |entry|
+		action_uri, action_file = cache_file(entry["action"])
+		metadata = nil
+		provided_files = []
+		missing_files = []
+		file.puts "\t# #{entry["id"]}"
+		file.puts "\t# #{entry["comment"]}"
+		file.puts "\tScenario: #{entry["id"]} #{entry["name"].gsub("<", "less than")}"
+		if entry["action"].end_with?(".json")
+			file.puts "\t\tGiven I have a metadata file called \"csvw/#{entry["action"]}\""
+			file.puts "\t\tAnd the metadata is stored at the url \"#{action_uri}\""
+		else
+			file.puts "\t\tGiven I have a CSV file called \"csvw/#{entry["action"]}\""
+			file.puts "\t\tAnd it has a Link header holding \"#{entry["httpLink"]}\"" if entry["httpLink"]
+			file.puts "\t\tAnd it is stored at the url \"#{action_uri}\""
+			if entry["option"] && entry["option"]["metadata"]
+				# no need to store the file here, as it will be listed in the 'implicit' list, which all get stored
+				metadata = URI.join(BASE_URI, entry["option"]["metadata"])
+				file.puts "\t\tAnd I have a metadata file called \"csvw/#{entry["option"]["metadata"]}\""
+				file.puts "\t\tAnd the metadata is stored at the url \"#{metadata}\""
+			end
+			provided_files << action_uri.to_s
+			missing_files = [				  
+  				URI.join(action_uri, '/.well-known/csvm').to_s,
+  				"#{action_uri}-metadata.json",
+  				URI.join(action_uri, 'csv-metadata.json').to_s
+			]
+		end
+		entry["implicit"].each do |implicit|
+			implicit_uri, implicit_file = cache_file(implicit)
+			provided_files << implicit_uri.to_s
+			unless implicit_uri == metadata
+				file.puts "\t\tAnd I have a file called \"csvw/#{implicit}\" at the url \"#{implicit_uri}\""
+			end
+		end if entry["implicit"]
+		missing_files.each do |uri|
+			file.puts "\t\tAnd there is no file at the url \"#{uri}\"" unless provided_files.include? uri
+		end
+		file.puts "\t\tWhen I carry out CSVW validation"
+		if entry["type"] == "csvt:WarningValidationTest"
+			file.puts "\t\tThen there should not be errors"
+			file.puts "\t\tAnd there should be warnings"
+		elsif entry["type"] == "csvt:NegativeValidationTest"
+	    file.puts "\t\tThen there should be errors"
+		else
+	    file.puts "\t\tThen there should not be errors"
+	    file.puts "\t\tAnd there should not be warnings"
+		end
+		file.puts "\t"
+	end
+end unless File.exist? FEATURE_FILE_PATH

data/features/validation_errors.feature

@@ -26,7 +26,7 @@ Feature: Get validation errors
     And that error should have the type "unclosed_quote"
     And that error should have the row "2"
     And that error should have the content ""Foo","Bar","Baz"
-    
+
    Scenario: Successfully report a CSV with incorrect whitespace
     Given I have a CSV with the following content:
     """
@@ -39,7 +39,7 @@ Feature: Get validation errors
     And that error should have the type "whitespace"
     And that error should have the row "2"
     And that error should have the content ""Foo","Bar",   "Baz""
-    
+
   Scenario: Successfully report a CSV with blank rows
     Given I have a CSV with the following content:
     """
@@ -83,40 +83,28 @@ Feature: Get validation errors
     Then there should be 1 error
     And that error should have the type "blank_rows"
     And that error should have the row "3"
-    
+
   Scenario: Report invalid Encoding
     Given I have a CSV file called "invalid-byte-sequence.csv"
     And I set an encoding header of "UTF-8"
     And it is stored at the url "http://example.com/example1.csv"
     When I ask if there are errors
-    Then there should be 1 error    
+    Then there should be 1 error
     And that error should have the type "invalid_encoding"
-    
+
   Scenario: Correctly handle different encodings
     Given I have a CSV file called "invalid-byte-sequence.csv"
-    And I set an encoding header of "ISO-8859-1"    
+    And I set an encoding header of "ISO-8859-1"
     And it is stored at the url "http://example.com/example1.csv"
     When I ask if there are errors
-    Then there should be no "content_encoding" errors  
-    
+    Then there should be no "content_encoding" errors
+
   Scenario: Report invalid file
-    
     Given I have a CSV file called "spreadsheet.xls"
     And it is stored at the url "http://example.com/example1.csv"
     When I ask if there are errors
-    Then there should be 1 error    
-    And that error should have the type "invalid_encoding"
-    
-  Scenario: Incorrect content type
-    Given I have a CSV with the following content:
-    """
-"abc","2","3"
-    """
-    And the content type is set to "application/excel"
-    And it is stored at the url "http://example.com/example1.xls"
-    And I ask if there are errors
     Then there should be 1 error
-    And that error should have the type "wrong_content_type"
+    And that error should have the type "invalid_encoding"
 
   Scenario: Incorrect extension
     Given I have a CSV with the following content:
@@ -128,13 +116,13 @@ Feature: Get validation errors
     And I ask if there are errors
     Then there should be 1 error
     And that error should have the type "wrong_content_type"
-    
+
   Scenario: Handles urls that 404
     Given I have a CSV that doesn't exist
     When I ask if there are errors
     Then there should be 1 error
     And that error should have the type "not_found"
-    
+
   Scenario: Incorrect line endings specified in settings
     Given I have a CSV file called "cr-line-endings.csv"
     And I set the line endings to linefeed
@@ -142,7 +130,7 @@ Feature: Get validation errors
     And I ask if there are errors
     Then there should be 1 error
     And that error should have the type "line_breaks"
-  
+
   Scenario: inconsistent line endings in file cause an error
     Given I have a CSV file called "inconsistent-line-endings.csv"
     And it is stored at the url "http://example.com/example1.csv"

data/features/validation_warnings.feature

@@ -10,18 +10,18 @@ Feature: Validation warnings
     And it is stored at the url "http://example.com/example1.csv"
     When I ask if there are warnings
     Then there should be 0 warnings
-    
+
    Scenario: ISO-8859-1 Encoding
     Given I have a CSV with the following content:
     """
 "col1","col2","col3"
 "1","2","3"
     """
-    And it is encoded as "iso-8859-1"    
+     And it is encoded as "iso-8859-1"
     And it is stored at the url "http://example.com/example1.csv"
     When I ask if there are warnings
     Then there should be 1 warnings
-  
+
   Scenario: Correct content type
     Given I have a CSV with the following content:
     """
@@ -32,7 +32,7 @@ Feature: Validation warnings
     And it is stored at the url "http://example.com/example1.csv"
     And I ask if there are warnings
     Then there should be 0 warnings
-    
+
   Scenario: No extension
     Given I have a CSV with the following content:
     """
@@ -43,7 +43,7 @@ Feature: Validation warnings
     And it is stored at the url "http://example.com/example1"
     And I ask if there are warnings
     Then there should be 0 warnings
-    
+
   Scenario: Allow query params after extension
     Given I have a CSV with the following content:
     """
@@ -54,7 +54,7 @@ Feature: Validation warnings
     And it is stored at the url "http://example.com/example1.csv?query=param"
     And I ask if there are warnings
     Then there should be 0 warnings
-    
+
   Scenario: User doesn't supply encoding
     Given I have a CSV with the following content:
     """
@@ -72,3 +72,15 @@ Feature: Validation warnings
     And I ask if there are warnings
     Then there should be 1 warnings
     And that warning should have the type "title_row"
+
+  Scenario: catch excel warnings
+    Given I parse a file called "spreadsheet.xls"
+    And I ask if there are warnings
+    Then there should be 1 warnings
+    And that warning should have the type "excel"
+
+  Scenario: catch excel warnings
+    Given I parse a file called "spreadsheet.xlsx"
+    And I ask if there are warnings
+    Then there should be 1 warnings
+    And that warning should have the type "excel"