wjordan213.csvlint 0.2.8

data/features/information.feature

@@ -0,0 +1,22 @@
+Feature: Return information
+
+  Background:
+    Given I have a CSV with the following content:
+    """
+"abc","2","3"
+    """
+    And it is encoded as "utf-8"
+    And the content type is "text/csv"
+    And it is stored at the url "http://example.com/example1.csv?query=true"
+
+  Scenario: Return encoding
+    Then the "encoding" should be "UTF-8"
+
+  Scenario: Return content type
+    Then the "content_type" should be "text/csv; charset=utf-8"
+
+  Scenario: Return extension
+    Then the "extension" should be ".csv"
+
+  Scenario: Return meta
+    Then the metadata content type should be "text/csv; charset=utf-8"

data/features/parse_csv.feature

@@ -0,0 +1,90 @@
+Feature: Parse CSV
+
+  Scenario: Successfully parse a valid CSV
+    Given I have a CSV with the following content:
+    """
+"Foo","Bar","Baz"
+"1","2","3"
+"3","2","1"
+    """
+    And it is stored at the url "http://example.com/example1.csv"
+    When I ask if the CSV is valid
+    Then I should get the value of true
+
+   Scenario: Successfully parse a CSV with newlines in quoted fields
+    Given I have a CSV with the following content:
+    """
+"a","b","c"
+"d","e","this is
+valid"
+"a","b","c"
+"""
+    And it is stored at the url "http://example.com/example1.csv"
+    When I ask if the CSV is valid
+    Then I should get the value of true
+
+   Scenario: Successfully parse a CSV with multiple newlines in quoted fields
+    Given I have a CSV with the following content:
+    """
+"a","b","c"
+"d","this is
+valid","as is this
+too"
+"""
+    And it is stored at the url "http://example.com/example1.csv"
+    When I ask if the CSV is valid
+    Then I should get the value of true
+
+  Scenario: Successfully report an invalid CSV
+    Given I have a CSV with the following content:
+    """
+	"Foo",	"Bar"	,	"Baz
+    """
+    And it is stored at the url "http://example.com/example1.csv"
+    When I ask if the CSV is valid
+    Then I should get the value of false
+
+   Scenario: Successfully report a CSV with incorrect quoting
+    Given I have a CSV with the following content:
+    """
+"Foo","Bar","Baz
+    """
+    And it is stored at the url "http://example.com/example1.csv"
+    When I ask if the CSV is valid
+    Then I should get the value of false
+
+   Scenario: Successfully report a CSV with incorrect whitespace
+    Given I have a CSV with the following content:
+    """
+"Foo","Bar",   "Baz"
+    """
+    And it is stored at the url "http://example.com/example1.csv"
+    When I ask if the CSV is valid
+    Then I should get the value of false
+
+   Scenario: Successfully report a CSV with ragged rows
+    Given I have a CSV with the following content:
+    """
+"col1","col2","col2"
+"1","2","3"
+"4","5"
+    """
+    And it is stored at the url "http://example.com/example1.csv"
+    When I ask if the CSV is valid
+    Then I should get the value of false
+
+    Scenario: Don't class blank values as inconsistencies
+     Given I have a CSV with the following content:
+     """
+"col1","col2","col3"
+"1","2","3"
+"4","5","6"
+"","7","8"
+"9","10","11"
+"","12","13"
+"","14","15"
+"16","17","18"
+     """
+     And it is stored at the url "http://example.com/example1.csv"
+     When I ask if there are warnings
+     Then there should be 0 warnings

data/features/schema_validation.feature

@@ -0,0 +1,105 @@
+Feature: Schema Validation
+
+  Scenario: Valid CSV
+    Given I have a CSV with the following content:
+    """
+"Bob","1234","bob@example.org"
+"Alice","5","alice@example.com"
+    """
+    And it is stored at the url "http://example.com/example1.csv"
+    And I have a schema with the following content:
+    """
+{
+	"fields": [
+          { "name": "Name", "constraints": { "required": true } },
+          { "name": "Id", "constraints": { "required": true, "minLength": 1 } },
+          { "name": "Email", "constraints": { "required": true } }
+    ]
+}
+    """
+    When I ask if there are errors
+    Then there should be 0 error
+
+  Scenario: Schema invalid CSV
+    Given I have a CSV with the following content:
+    """
+"Bob","1234","bob@example.org"
+"Alice","5","alice@example.com"
+    """
+    And it is stored at the url "http://example.com/example1.csv"
+    And I have a schema with the following content:
+    """
+{
+	"fields": [
+          { "name": "Name", "constraints": { "required": true } },
+          { "name": "Id", "constraints": { "required": true, "minLength": 3 } },
+          { "name": "Email", "constraints": { "required": true } }
+    ]
+}
+    """
+    When I ask if there are errors
+    Then there should be 1 error
+
+  Scenario: CSV with incorrect header
+    Given I have a CSV with the following content:
+    """
+"name","id","contact"
+"Bob","1234","bob@example.org"
+"Alice","5","alice@example.com"
+    """
+    And it is stored at the url "http://example.com/example1.csv"
+    And I have a schema with the following content:
+    """
+{
+	"fields": [
+          { "name": "name", "constraints": { "required": true } },
+          { "name": "id", "constraints": { "required": true, "minLength": 3 } },
+          { "name": "email", "constraints": { "required": true } }
+    ]
+}
+    """
+    When I ask if there are warnings
+    Then there should be 1 warnings
+
+  Scenario: Schema with valid regex
+    Given I have a CSV with the following content:
+    """
+  "firstname","id","email"
+  "Bob","1234","bob@example.org"
+  "Alice","5","alice@example.com"
+    """
+    And it is stored at the url "http://example.com/example1.csv"
+    And I have a schema with the following content:
+    """
+{
+  "fields": [
+          { "name": "Name", "constraints": { "required": true, "pattern": "^[A-Za-z0-9_]*$" } },
+          { "name": "Id", "constraints": { "required": true, "minLength": 1 } },
+          { "name": "Email", "constraints": { "required": true } }
+    ]
+}
+    """
+    When I ask if there are errors
+    Then there should be 0 error
+
+  Scenario: Schema with invalid regex
+    Given I have a CSV with the following content:
+    """
+  "firstname","id","email"
+  "Bob","1234","bob@example.org"
+  "Alice","5","alice@example.com"
+    """
+    And it is stored at the url "http://example.com/example1.csv"
+    And I have a schema with the following content:
+    """
+{
+  "fields": [
+          { "name": "Name", "constraints": { "required": true, "pattern": "((" } },
+          { "name": "Id", "constraints": { "required": true, "minLength": 1 } },
+          { "name": "Email", "constraints": { "required": true } }
+    ]
+}
+    """
+    When I ask if there are errors
+    Then there should be 1 error
+    And that error should have the type "invalid_regex"

data/features/sources.feature

@@ -0,0 +1,17 @@
+Feature: Parse CSV from Different Sources
+
+  Scenario: Successfully parse a valid CSV from a StringIO
+      Given I have a CSV with the following content:
+    """
+"Foo","Bar","Baz"
+"1","2","3"
+"3","2","1"
+    """
+    And it is parsed as a StringIO
+    When I ask if the CSV is valid
+    Then I should get the value of true
+
+  Scenario: Successfully parse a valid CSV from a File
+    Given I parse a file called "valid.csv"
+    When I ask if the CSV is valid
+    Then I should get the value of true

data/features/step_definitions/cli_steps.rb

@@ -0,0 +1,11 @@
+Given(/^I have stubbed stdin to contain "(.*?)"$/) do |file|
+  expect(STDIN).to receive(:read).and_return(File.read(file))
+end
+
+Given(/^I have stubbed stdin to contain nothing$/) do
+  expect(STDIN).to receive(:read).and_return(nil)
+end
+
+Then(/^nothing should be outputted to STDERR$/) do
+  expect(STDERR).to_not receive(:puts)
+end

data/features/step_definitions/csv_options_steps.rb

@@ -0,0 +1,24 @@
+Given(/^I set the delimiter to "(.*?)"$/) do |delimiter|
+  @csv_options ||= default_csv_options
+  @csv_options["delimiter"] = delimiter
+end
+
+Given(/^I set quotechar to "(.*?)"$/) do |doublequote|
+  @csv_options ||= default_csv_options
+  @csv_options["quoteChar"] = doublequote
+end
+
+Given(/^I set the line endings to linefeed$/) do
+  @csv_options ||= default_csv_options
+  @csv_options["lineTerminator"] = "\n"
+end
+
+Given(/^I set the line endings to carriage return$/) do
+  @csv_options ||= default_csv_options
+  @csv_options["lineTerminator"] = "\r"
+end
+
+Given(/^I set header to "(.*?)"$/) do |boolean|
+  @csv_options ||= default_csv_options
+  @csv_options["header"] = boolean == "true"
+end

data/features/step_definitions/information_steps.rb

@@ -0,0 +1,13 @@
+Given(/^the content type is "(.*?)"$/) do |arg1|
+  @content_type = "text/csv"
+end
+
+Then(/^the "(.*?)" should be "(.*?)"$/) do |type, encoding|
+  validator = Csvlint::Validator.new( @url, default_csv_options ) 
+  expect( validator.send(type.to_sym) ).to eq( encoding )
+end
+
+Then(/^the metadata content type should be "(.*?)"$/) do |content_type|
+  validator = Csvlint::Validator.new( @url, default_csv_options ) 
+  expect( validator.headers['content-type'] ).to eq( content_type )
+end

data/features/step_definitions/parse_csv_steps.rb

@@ -0,0 +1,42 @@
+Given(/^I have a CSV with the following content:$/) do |string|
+  @csv = string.to_s
+end
+
+Given(/^it has a Link header holding "(.*?)"$/) do |link|
+  @link = "#{link}; type=\"application/csvm+json\""
+end
+
+Given(/^it is stored at the url "(.*?)"$/) do |url|
+  @url = url
+  content_type = @content_type || "text/csv"
+  charset = @encoding || "UTF-8"
+  headers = {"Content-Type" => "#{content_type}; charset=#{charset}"}
+  headers["Link"] = @link if @link
+  stub_request(:get, url).to_return(:status => 200, :body => @csv, :headers => headers)
+  stub_request(:get, URI.join(url, '/.well-known/csvm')).to_return(:status => 404)
+  stub_request(:get, url + '-metadata.json').to_return(:status => 404)
+  stub_request(:get, URI.join(url, 'csv-metadata.json')).to_return(:status => 404)
+end
+
+Given(/^it is stored at the url "(.*?)" with no character set$/) do |url|
+  @url = url
+  content_type = @content_type || "text/csv"
+  stub_request(:get, url).to_return(:status => 200, :body => @csv, :headers => {"Content-Type" => "#{content_type}"})
+  stub_request(:get, URI.join(url, '/.well-known/csvm')).to_return(:status => 404)
+  stub_request(:get, url + '-metadata.json').to_return(:status => 404)
+  stub_request(:get, URI.join(url, 'csv-metadata.json')).to_return(:status => 404)
+end
+
+When(/^I ask if the CSV is valid$/) do
+  @csv_options ||= default_csv_options
+  @validator = Csvlint::Validator.new( @url, @csv_options )
+  @valid = @validator.valid?
+end
+
+Then(/^I should get the value of true$/) do
+  expect( @valid ).to be(true)
+end
+
+Then(/^I should get the value of false$/) do
+  expect( @valid ).to be(false)
+end

data/features/step_definitions/schema_validation_steps.rb

@@ -0,0 +1,33 @@
+Given(/^I have a schema with the following content:$/) do |json|
+  @schema_type = :json_table
+  @schema_json = json
+end
+
+Given(/^I have metadata with the following content:$/) do |json|
+  @schema_type = :csvw_metadata
+  @schema_json = json
+end
+
+Given(/^I have a metadata file called "([^"]*)"$/) do |filename|
+  @schema_type = :csvw_metadata
+  @schema_json = File.read( File.join( File.dirname(__FILE__), "..", "fixtures", filename ) )
+end
+
+Given(/^the (schema|metadata) is stored at the url "(.*?)"$/) do |schema_type,schema_url|
+  @schema_url = schema_url
+  stub_request(:get, @schema_url).to_return(:status => 200, :body => @schema_json.to_str)
+end
+
+Given(/^there is a file at "(.*?)" with the content:$/) do |url, content|
+  stub_request(:get, url).to_return(:status => 200, :body => content.to_str)
+end
+
+Given(/^I have a file called "(.*?)" at the url "(.*?)"$/) do |filename,url|
+  content = File.read( File.join( File.dirname(__FILE__), "..", "fixtures", filename ) )
+  content_type = filename =~ /.csv$/ ? "text/csv" : "application/csvm+json"
+  stub_request(:get, url).to_return(:status => 200, :body => content, :headers => {"Content-Type" => "#{content_type}; charset=UTF-8"})
+end
+
+Given(/^there is no file at the url "(.*?)"$/) do |url|
+  stub_request(:get, url).to_return(:status => 404)
+end

data/features/step_definitions/sources_steps.rb

@@ -0,0 +1,7 @@
+Given(/^it is parsed as a StringIO$/) do
+  @url = StringIO.new(@csv)
+end
+
+Given(/^I parse a file called "(.*?)"$/) do |filename|
+  @url = File.new( File.join( File.dirname(__FILE__), "..", "fixtures", filename ) )
+end

data/features/step_definitions/validation_errors_steps.rb

@@ -0,0 +1,90 @@
+When(/^I ask if there are errors$/) do
+  @csv_options ||= default_csv_options
+
+  if @schema_json
+    if @schema_type == :json_table
+      @schema = Csvlint::Schema.from_json_table( @schema_url || "http://example.org ", JSON.parse(@schema_json) )
+    else
+      @schema = Csvlint::Schema.from_csvw_metadata( @schema_url || "http://example.org ", JSON.parse(@schema_json) )
+    end
+  end
+
+  @validator = Csvlint::Validator.new( @url, @csv_options, @schema )
+  @errors = @validator.errors
+end
+
+When(/^I carry out CSVW validation$/) do
+  @csv_options ||= default_csv_options
+
+  begin
+    if @schema_json
+      json = JSON.parse(@schema_json)
+      if @schema_type == :json_table
+        @schema = Csvlint::Schema.from_json_table( @schema_url || "http://example.org ", json )
+      else
+        @schema = Csvlint::Schema.from_csvw_metadata( @schema_url || "http://example.org ", json )
+      end
+    end
+
+    if @url.nil?
+      @errors = []
+      @warnings = []
+      @schema.tables.keys.each do |table_url|
+        validator = Csvlint::Validator.new( table_url, @csv_options, @schema )
+        @errors += validator.errors
+        @warnings += validator.warnings
+      end
+    else
+      validator = Csvlint::Validator.new( @url, @csv_options, @schema )
+      @errors = validator.errors
+      @warnings = validator.warnings
+    end
+  rescue JSON::ParserError => e
+    @errors = [e]
+  rescue Csvlint::Csvw::MetadataError => e
+    @errors = [e]
+  end
+end
+
+Then(/^there should be errors$/) do
+  # this test is only used for CSVW testing; :invalid_encoding masks lack of real errors
+  @errors.delete_if { |e| e.instance_of?(Csvlint::ErrorMessage) && [:invalid_encoding, :line_breaks].include?(e.type) }
+  expect( @errors.count ).to be > 0
+end
+
+Then(/^there should not be errors$/) do
+  expect( @errors.count ).to eq(0)
+end
+
+Then(/^there should be (\d+) error$/) do |count|
+  expect( @errors.count ).to eq( count.to_i )
+end
+
+Then(/^that error should have the type "(.*?)"$/) do |type|
+  expect( @errors.first.type ).to eq( type.to_sym )
+end
+
+Then(/^that error should have the row "(.*?)"$/) do |row|
+  expect( @errors.first.row ).to eq( row.to_i )
+end
+
+Then(/^that error should have the column "(.*?)"$/) do |column|
+  expect( @errors.first.column ).to eq( column.to_i )
+end
+
+Then(/^that error should have the content "(.*)"$/) do |content|
+  expect( @errors.first.content.chomp ).to eq( content.chomp )
+end
+
+Then(/^that error should have no content$/) do
+  expect( @errors.first.content ).to eq( nil )
+end
+
+Given(/^I have a CSV that doesn't exist$/) do
+  @url = "http//www.example.com/fake-csv.csv"
+  stub_request(:get, @url).to_return(:status => 404)
+end
+
+Then(/^there should be no "(.*?)" errors$/) do |type|
+  @errors.each do |error| error.type.should_not == type.to_sym end
+end