wjordan213-csvlint 0.2.8

data/features/step_definitions/validation_info_steps.rb

@@ -0,0 +1,22 @@
+Given(/^I ask if there are info messages$/) do
+  @csv_options ||= default_csv_options
+  
+  if @schema_json
+    if @schema_type == :json_table
+      @schema = Csvlint::Schema.from_json_table( @schema_url || "http://example.org ", JSON.parse(@schema_json) )
+    else
+      @schema = Csvlint::Schema.from_csvw_metadata( @schema_url || "http://example.org ", JSON.parse(@schema_json) )
+    end
+  end
+  
+  @validator = Csvlint::Validator.new( @url, @csv_options, @schema ) 
+  @info_messages = @validator.info_messages
+end
+
+Then(/^there should be (\d+) info messages?$/) do |num|
+  expect( @info_messages.count ).to eq( num.to_i )
+end
+
+Then(/^one of the messages should have the type "(.*?)"$/) do |msg_type|
+  expect( @info_messages.find{|x| x.type == msg_type.to_sym} ).to be_present
+end

data/features/step_definitions/validation_warnings_steps.rb

@@ -0,0 +1,60 @@
+Given(/^it is encoded as "(.*?)"$/) do |encoding|
+  @csv = @csv.encode(encoding)
+  @encoding = encoding
+end
+
+Given(/^I set an encoding header of "(.*?)"$/) do |encoding|
+  @encoding = encoding
+end
+
+Given(/^I do not set an encoding header$/) do
+  @encoding = nil
+end
+
+Given(/^I have a CSV file called "(.*?)"$/) do |filename|
+  @csv = File.read( File.join( File.dirname(__FILE__), "..", "fixtures", filename ) )
+end
+
+When(/^I ask if there are warnings$/) do
+  @csv_options ||= default_csv_options
+  if @schema_json
+    if @schema_type == :json_table
+      @schema = Csvlint::Schema.from_json_table( @schema_url || "http://example.org ", JSON.parse(@schema_json) )
+    else
+      @schema = Csvlint::Schema.from_csvw_metadata( @schema_url || "http://example.org ", JSON.parse(@schema_json) )
+    end
+  end
+
+  @validator = Csvlint::Validator.new( @url, @csv_options, @schema )
+  @warnings = @validator.warnings
+end
+
+Then(/^there should be warnings$/) do
+  expect( @warnings.count ).to be > 0
+end
+
+Then(/^there should not be warnings$/) do
+  # this test is only used for CSVW testing, and :inconsistent_values warnings don't count in CSVW
+  @warnings.delete_if { |w| [:inconsistent_values, :check_options].include?(w.type) }
+  expect( @warnings.count ).to eq(0)
+end
+
+Then(/^there should be (\d+) warnings$/) do |count|
+  expect( @warnings.count ).to eq( count.to_i )
+end
+
+Given(/^the content type is set to "(.*?)"$/) do |type|
+  @content_type = type
+end
+
+Then(/^that warning should have the row "(.*?)"$/) do |row|
+  expect( @warnings.first.row ).to eq( row.to_i )
+end
+
+Then(/^that warning should have the column "(.*?)"$/) do |column|
+  expect( @warnings.first.column ).to eq( column.to_i )
+end
+
+Then(/^that warning should have the type "(.*?)"$/) do |type|
+  expect( @warnings.first.type ).to eq( type.to_sym )
+end

data/features/support/aruba.rb

@@ -0,0 +1,56 @@
+require 'aruba'
+require 'aruba/in_process'
+require 'aruba/cucumber'
+
+require 'csvlint/cli'
+
+module Csvlint
+  class CliRunner
+    # Allow everything fun to be injected from the outside while defaulting to normal implementations.
+    def initialize(argv, stdin = STDIN, stdout = STDOUT, stderr = STDERR, kernel = Kernel)
+      @argv, @stdin, @stdout, @stderr, @kernel = argv, stdin, stdout, stderr, kernel
+    end
+
+    def execute!
+      exit_code = begin
+        # Thor accesses these streams directly rather than letting them be injected, so we replace them...
+        $stderr = @stderr
+        $stdin = @stdin
+        $stdout = @stdout
+
+        # Run our normal Thor app the way we know and love.
+        Csvlint::Cli.start(@argv.dup.unshift("validate"))
+
+        # Thor::Base#start does not have a return value, assume success if no exception is raised.
+        0
+      rescue StandardError => e
+        # The ruby interpreter would pipe this to STDERR and exit 1 in the case of an unhandled exception
+        b = e.backtrace
+        @stderr.puts("#{b.shift}: #{e.message} (#{e.class})")
+        @stderr.puts(b.map{|s| "\tfrom #{s}"}.join("\n"))
+        1
+      rescue SystemExit => e
+        e.status
+      ensure
+        # TODO: reset your app here, free up resources, etc.
+        # Examples:
+        # MyApp.logger.flush
+        # MyApp.logger.close
+        # MyApp.logger = nil
+        #
+        # MyApp.reset_singleton_instance_variables
+
+        # ...then we put the streams back.
+        $stderr = STDERR
+        $stdin = STDIN
+        $stdout = STDOUT
+      end
+
+      # Proxy our exit code back to the injected kernel.
+      @kernel.exit(exit_code)
+    end
+  end
+end
+
+Aruba.process = Aruba::Processes::InProcess
+Aruba.process.main_class = Csvlint::CliRunner

data/features/support/env.rb

@@ -0,0 +1,26 @@
+require 'coveralls'
+Coveralls.wear_merged!('test_frameworks')
+
+$:.unshift File.join( File.dirname(__FILE__), "..", "..", "lib")
+
+require 'rspec/expectations'
+require 'cucumber/rspec/doubles'
+require 'csvlint'
+require 'pry'
+
+require 'spork'
+
+Spork.each_run do
+  require 'csvlint'
+end
+
+class CustomWorld
+  def default_csv_options
+    return {
+    }
+  end
+end
+
+World do
+  CustomWorld.new
+end

data/features/support/load_tests.rb

@@ -0,0 +1,114 @@
+require 'json'
+require 'open-uri'
+require 'uri'
+
+BASE_URI = "http://w3c.github.io/csvw/tests/"
+BASE_PATH = File.join(File.dirname(__FILE__), "..", "fixtures", "csvw")
+FEATURE_FILE_PATH = File.join(File.dirname(__FILE__), "..", "csvw_validation_tests.feature")
+SCRIPT_FILE_PATH = File.join(File.dirname(__FILE__), "..", "..", "bin", "run-csvw-tests")
+
+Dir.mkdir(BASE_PATH) unless Dir.exist?(BASE_PATH)
+
+def cache_file(filename)
+	file = File.join(BASE_PATH, filename)
+	uri = URI.join(BASE_URI, filename)
+	unless File.exist?(file)
+		if filename.include? "/"
+			levels = filename.split("/")[0..-2]
+			for i in 0..levels.length
+				dir = File.join(BASE_PATH, levels[0..i].join("/"))
+				Dir.mkdir(dir) unless Dir.exist?(dir)
+			end
+		end
+		STDERR.puts("storing #{file} locally")
+		File.open(file, 'wb') do |f|
+			f.puts open(uri, 'rb').read
+		end
+	end
+	return uri, file
+end
+
+File.open(SCRIPT_FILE_PATH, 'w') do |file|
+	File.chmod(0755, SCRIPT_FILE_PATH)
+	manifest = JSON.parse( open("http://w3c.github.io/csvw/tests/manifest-validation.jsonld").read )
+	manifest["entries"].each do |entry|
+		type = "valid"
+		case entry["type"] 
+		when "csvt:WarningValidationTest" 
+			type = "warnings" 
+		when "csvt:NegativeValidationTest" 
+			type = "errors" 
+		end
+		file.puts "echo \"#{entry["id"].split("#")[-1]}: #{entry["name"].gsub("`", "'")}\""
+		file.puts "echo \"#{type}: #{entry["comment"].gsub("\"", "\\\"").gsub("`", "'")}\""
+		if entry["action"].end_with?(".json")
+			file.puts "csvlint --schema=features/fixtures/csvw/#{entry["action"]}"
+		elsif entry["option"] && entry["option"]["metadata"]
+			file.puts "csvlint features/fixtures/csvw/#{entry["action"]} --schema=features/fixtures/csvw/#{entry["option"]["metadata"]}"
+		else
+			file.puts "csvlint features/fixtures/csvw/#{entry["action"]}"
+		end
+		file.puts "echo"
+	end
+end unless File.exist? SCRIPT_FILE_PATH
+
+File.open(FEATURE_FILE_PATH, 'w') do |file|
+	file.puts "# Auto-generated file based on standard validation CSVW tests from http://w3c.github.io/csvw/tests/manifest-validation.jsonld"
+	file.puts ""
+
+	manifest = JSON.parse( open("http://w3c.github.io/csvw/tests/manifest-validation.jsonld").read )
+
+	file.puts "Feature: #{manifest["label"]}"
+	file.puts ""
+	
+	manifest["entries"].each do |entry|
+		action_uri, action_file = cache_file(entry["action"])
+		metadata = nil
+		provided_files = []
+		missing_files = []
+		file.puts "\t# #{entry["id"]}"
+		file.puts "\t# #{entry["comment"]}"
+		file.puts "\tScenario: #{entry["id"]} #{entry["name"].gsub("<", "less than")}"
+		if entry["action"].end_with?(".json")
+			file.puts "\t\tGiven I have a metadata file called \"csvw/#{entry["action"]}\""
+			file.puts "\t\tAnd the metadata is stored at the url \"#{action_uri}\""
+		else
+			file.puts "\t\tGiven I have a CSV file called \"csvw/#{entry["action"]}\""
+			file.puts "\t\tAnd it has a Link header holding \"#{entry["httpLink"]}\"" if entry["httpLink"]
+			file.puts "\t\tAnd it is stored at the url \"#{action_uri}\""
+			if entry["option"] && entry["option"]["metadata"]
+				# no need to store the file here, as it will be listed in the 'implicit' list, which all get stored
+				metadata = URI.join(BASE_URI, entry["option"]["metadata"])
+				file.puts "\t\tAnd I have a metadata file called \"csvw/#{entry["option"]["metadata"]}\""
+				file.puts "\t\tAnd the metadata is stored at the url \"#{metadata}\""
+			end
+			provided_files << action_uri.to_s
+			missing_files = [				  
+  				URI.join(action_uri, '/.well-known/csvm').to_s,
+  				"#{action_uri}-metadata.json",
+  				URI.join(action_uri, 'csv-metadata.json').to_s
+			]
+		end
+		entry["implicit"].each do |implicit|
+			implicit_uri, implicit_file = cache_file(implicit)
+			provided_files << implicit_uri.to_s
+			unless implicit_uri == metadata
+				file.puts "\t\tAnd I have a file called \"csvw/#{implicit}\" at the url \"#{implicit_uri}\""
+			end
+		end if entry["implicit"]
+		missing_files.each do |uri|
+			file.puts "\t\tAnd there is no file at the url \"#{uri}\"" unless provided_files.include? uri
+		end
+		file.puts "\t\tWhen I carry out CSVW validation"
+		if entry["type"] == "csvt:WarningValidationTest"
+			file.puts "\t\tThen there should not be errors"
+			file.puts "\t\tAnd there should be warnings"
+		elsif entry["type"] == "csvt:NegativeValidationTest"
+	    file.puts "\t\tThen there should be errors"
+		else
+	    file.puts "\t\tThen there should not be errors"
+	    file.puts "\t\tAnd there should not be warnings"
+		end
+		file.puts "\t"
+	end
+end unless File.exist? FEATURE_FILE_PATH

data/features/support/webmock.rb

@@ -0,0 +1 @@
+require 'webmock/cucumber'

data/features/validation_errors.feature

@@ -0,0 +1,147 @@
+Feature: Get validation errors
+
+  Scenario: CSV with ragged rows
+    Given I have a CSV with the following content:
+    """
+"col1","col2","col3"
+"1","2","3"
+"4","5"
+    """
+    And it is stored at the url "http://example.com/example1.csv"
+    When I ask if there are errors
+    Then there should be 1 error
+    And that error should have the type "ragged_rows"
+    And that error should have the row "3"
+    And that error should have the content ""4","5""
+
+  Scenario: CSV with incorrect quoting
+    Given I have a CSV with the following content:
+    """
+"col1","col2","col3"
+"Foo","Bar","Baz
+    """
+    And it is stored at the url "http://example.com/example1.csv"
+    When I ask if there are errors
+    Then there should be 1 error
+    And that error should have the type "unclosed_quote"
+    And that error should have the row "2"
+    And that error should have the content ""Foo","Bar","Baz"
+
+  Scenario: Successfully report a CSV with incorrect whitespace
+    Given I have a CSV with the following content:
+    """
+"col1","col2","col3"
+"Foo","Bar",   "Baz"
+    """
+    And it is stored at the url "http://example.com/example1.csv"
+    When I ask if there are errors
+    Then there should be 1 error
+    And that error should have the type "whitespace"
+    And that error should have the row "2"
+    And that error should have the content ""Foo","Bar",   "Baz""
+
+  Scenario: Successfully report a CSV with blank rows
+    Given I have a CSV with the following content:
+    """
+"col1","col2","col3"
+"Foo","Bar","Baz"
+"","",
+"Baz","Bar","Foo"
+    """
+    And it is stored at the url "http://example.com/example1.csv"
+    When I ask if there are errors
+    Then there should be 1 error
+    And that error should have the type "blank_rows"
+    And that error should have the row "3"
+    And that error should have the content ""","","
+
+  Scenario: Successfully report a CSV with multiple trailing empty rows
+    Given I have a CSV with the following content:
+    """
+"col1","col2","col3"
+"Foo","Bar","Baz"
+"Foo","Bar","Baz"
+
+
+    """
+    And it is stored at the url "http://example.com/example1.csv"
+    When I ask if there are errors
+    Then there should be 1 error
+    And that error should have the type "blank_rows"
+    And that error should have the row "4"
+
+  Scenario: Successfully report a CSV with an empty row
+    Given I have a CSV with the following content:
+    """
+"col1","col2","col3"
+"Foo","Bar","Baz"
+
+"Foo","Bar","Baz"
+    """
+    And it is stored at the url "http://example.com/example1.csv"
+    When I ask if there are errors
+    Then there should be 1 error
+    And that error should have the type "blank_rows"
+    And that error should have the row "3"
+
+  Scenario: Report invalid Encoding
+    Given I have a CSV file called "invalid-byte-sequence.csv"
+    And I set an encoding header of "UTF-8"
+    And it is stored at the url "http://example.com/example1.csv"
+    When I ask if there are errors
+    Then there should be 1 error
+    And that error should have the type "invalid_encoding"
+
+  Scenario: Correctly handle different encodings
+    Given I have a CSV file called "invalid-byte-sequence.csv"
+    And I set an encoding header of "ISO-8859-1"
+    And it is stored at the url "http://example.com/example1.csv"
+    When I ask if there are errors
+    Then there should be no "content_encoding" errors
+
+  Scenario: Report invalid file
+    Given I have a CSV file called "spreadsheet.xls"
+    And it is stored at the url "http://example.com/example1.csv"
+    When I ask if there are errors
+    Then there should be 1 error
+    And that error should have the type "invalid_encoding"
+
+  Scenario: Incorrect extension
+    Given I have a CSV with the following content:
+    """
+"abc","2","3"
+    """
+    And the content type is set to "application/excel"
+    And it is stored at the url "http://example.com/example1.csv"
+    And I ask if there are errors
+    Then there should be 1 error
+    And that error should have the type "wrong_content_type"
+
+  Scenario: Handles urls that 404
+    Given I have a CSV that doesn't exist
+    When I ask if there are errors
+    Then there should be 1 error
+    And that error should have the type "not_found"
+
+  Scenario: Incorrect line endings specified in settings
+    Given I have a CSV file called "cr-line-endings.csv"
+    And I set the line endings to linefeed
+    And it is stored at the url "http://example.com/example1.csv"
+    And I ask if there are errors
+    Then there should be 1 error
+    And that error should have the type "line_breaks"
+
+  Scenario: inconsistent line endings in file cause an error
+    Given I have a CSV file called "inconsistent-line-endings.csv"
+    And it is stored at the url "http://example.com/example1.csv"
+    And I ask if there are errors
+    Then there should be 1 error
+    And that error should have the type "line_breaks"
+
+
+  Scenario: inconsistent line endings with unquoted fields in file cause an error
+    Given I have a CSV file called "inconsistent-line-endings-unquoted.csv"
+    And it is stored at the url "http://example.com/example1.csv"
+    And I ask if there are errors
+    Then there should be 1 error
+    And that error should have the type "line_breaks"

data/features/validation_info.feature

@@ -0,0 +1,16 @@
+Feature: Get validation information messages
+
+  Scenario: LF line endings in file give an info message
+    Given I have a CSV file called "lf-line-endings.csv"
+    And it is stored at the url "http://example.com/example1.csv"
+    And I set header to "true"
+    And I ask if there are info messages
+    Then there should be 1 info messages
+    And one of the messages should have the type "nonrfc_line_breaks"
+
+  Scenario: CRLF line endings in file produces no info messages
+    Given I have a CSV file called "crlf-line-endings.csv"
+    And it is stored at the url "http://example.com/example1.csv"
+    And I set header to "true"
+    And I ask if there are info messages
+    Then there should be 0 info messages