csvlint 0.0.1

data/features/csv_options.feature

@@ -0,0 +1,35 @@
+Feature: CSV options
+
+  Scenario: Sucessfully parse a valid CSV
+    Given I have a CSV with the following content:
+    """
+'Foo';'Bar';'Baz'
+'1';'2';'3'
+'3';'2';'1'
+    """
+    And I set the delimiter to ";"
+    And I set quotechar to "'" 
+    And it is stored at the url "http://example.com/example1.csv"
+    When I ask if the CSV is valid
+    Then I should get the value of true
+
+  Scenario: Warn if options seem to return invalid data
+    Given I have a CSV with the following content:
+    """
+'Foo';'Bar';'Baz'
+'1';'2';'3'
+'3';'2';'1'
+    """
+    And I set the delimiter to ","
+    And I set quotechar to """ 
+    And it is stored at the url "http://example.com/example1.csv"
+    And I ask if there are warnings
+    Then there should be 1 warnings
+    And that warning should have the type "check_options"
+
+  Scenario: Use esoteric line endings
+    Given I have a CSV file called "windows-line-endings.csv"
+    And it is stored at the url "http://example.com/example1.csv"
+    When I ask if the CSV is valid
+    Then I should get the value of true
+    

data/features/fixtures/cr-line-endings.csv

@@ -0,0 +1 @@
+"Foo","Bar","Baz"

data/features/fixtures/crlf-line-endings.csv

@@ -0,0 +1,3 @@
+"Foo","Bsr","Baz"
+"Biff","Baff","Boff"
+"Qux","Teaspoon","Doge"

data/features/fixtures/inconsistent-line-endings.csv

@@ -0,0 +1,2 @@
+"Foo","Bsr","Baz"
+"Qux","Teaspoon","Doge"

data/features/fixtures/invalid-byte-sequence.csv

@@ -0,0 +1,24 @@
+"Data","Dependencia Origem","Hist�rico","Data do Balancete","N�mero do documento","Valor",
+"10/31/2012","","Saldo Anterior","","0","100.00",
+"11/01/2012","0000-9","Transfer�ncia on line - 01/11 4885     256620-6 XXXXXXXXXXXXX","","224885000256620","100.00",
+"11/01/2012","","Dep�sito COMPE - 033 0502    27588602104 XXXXXXXXXXXXXX","","101150","100.00",
+"11/01/2012","","Proventos","","496774","1000.00",
+"11/01/2012","","Benef�cio","","496775","100.00",
+"11/01/2012","0000-0","Compra com Cart�o - 01/11 09:45 XXXXXXXXXXX","","135102","-1.00",
+"11/01/2012","0000-0","Compra com Cart�o - 01/11 09:48    XXXXXXXXXXX","","235338","-10.00",
+"11/01/2012","0000-0","Compra com Cart�o - 01/11 12:35 XXXXXXXX","","345329","-10.00",
+"11/01/2012","0000-0","Compra com Cart�o - 01/11 23:57    XXXXXXXXXXXXXXXX","","686249","-10.00",
+"11/01/2012","0000-0","Saque com cart�o - 01/11 13:17 XXXXXXXXXXXXXXXX","","11317296267021","-10.00",
+"11/01/2012","","Pagto conta telefone - VIVO DF","","110101","-100.00",
+"11/01/2012","","Cobran�a de I.O.F.","","391100701","-1.00",
+"11/05/2012","0000-0","Compra com Cart�o - 02/11 16:57    XXXXXXXXXXXX","","161057","-10.00",
+"11/05/2012","0000-0","Compra com Cart�o - 03/11 18:57    XXXXXXXXXXXXXXX","","168279","-10.00",
+"11/05/2012","0000-0","Compra com Cart�o - 05/11 12:32    XXXXXXXXXXXXXXXXX","","245166","-10.00",
+"11/05/2012","0000-0","Compra com Cart�o - 02/11 17:18    XXXXXXXXXXXXX","","262318","-1.00",
+"11/05/2012","0000-0","Compra com Cart�o - 02/11 22:46    XXXXXXXXXXX","","382002","-100.00",
+"11/05/2012","0000-0","Compra com Cart�o - 02/11 23:19    XXXXXXXXXXX","","683985","-1.00",
+"11/05/2012","0000-0","Compra com Cart�o - 03/11 01:19    XXXXXXXXXXXXXXXX","","704772","-10.00",
+"11/05/2012","0000-0","Compra com Cart�o - 03/11 11:08 XXXXXXXX","","840112","-1.00",
+"11/05/2012","0000-0","Saque com cart�o - 05/11 19:24 XXXXXXXXXXXXXXXXX","","51924256267021","-10.00",
+"11/05/2012","0000-0","Transfer�ncia on line - 05/11 4885     256620-6 XXXXXXXXXXXXX","","224885000256620","-100.00",
+"11/05/2012","","Pagamento de T�tulo - XXXXXXXXXXXXXXXXXXX","","110501","-100.00",

data/features/fixtures/lf-line-endings.csv

@@ -0,0 +1,3 @@
+"Foo","Bsr","Baz"
+"Biff","Baff","Boff"
+"Qux","Teaspoon","Doge"

data/features/fixtures/title-row.csv

@@ -0,0 +1,4 @@
+"This is a title row",,
+"Foo","Bsr","Baz"
+"Biff","Baff","Boff"
+"Qux","Teaspoon","Doge"

data/features/fixtures/valid.csv

@@ -0,0 +1,3 @@
+"Foo","Bar","Baz"
+"1","2","3"
+"3","2","1"

data/features/fixtures/windows-line-endings.csv

@@ -0,0 +1,2 @@
+a,b,c
+d,e,f

data/features/information.feature

@@ -0,0 +1,22 @@
+Feature: Return information
+
+  Background:
+    Given I have a CSV with the following content:
+    """
+"abc","2","3"
+    """
+    And it is encoded as "utf-8"
+    And the content type is "text/csv"
+    And it is stored at the url "http://example.com/example1.csv?query=true"
+
+  Scenario: Return encoding
+    Then the "encoding" should be "utf-8"
+    
+  Scenario: Return content type
+    Then the "content_type" should be "text/csv"
+
+  Scenario: Return extension
+    Then the "extension" should be ".csv"
+    
+  Scenario: Return meta
+    Then the metadata content type should be "text/csv; charset=utf-8"

data/features/parse_csv.feature

@@ -0,0 +1,90 @@
+Feature: Parse CSV
+  
+  Scenario: Sucessfully parse a valid CSV
+    Given I have a CSV with the following content:
+    """
+"Foo","Bar","Baz"
+"1","2","3"
+"3","2","1"
+    """
+    And it is stored at the url "http://example.com/example1.csv"
+    When I ask if the CSV is valid
+    Then I should get the value of true
+    
+   Scenario: Successfully parse a CSV with newlines in quoted fields
+    Given I have a CSV with the following content:
+    """
+"a","b","c"
+"d","e","this is 
+valid"
+"a","b","c"
+"""
+    And it is stored at the url "http://example.com/example1.csv"
+    When I ask if the CSV is valid
+    Then I should get the value of true
+
+   Scenario: Successfully parse a CSV with multiple newlines in quoted fields
+    Given I have a CSV with the following content:
+    """
+"a","b","c"
+"d","this is 
+valid","as is this 
+too"
+"""
+    And it is stored at the url "http://example.com/example1.csv"
+    When I ask if the CSV is valid
+    Then I should get the value of true
+        
+  Scenario: Successfully report an invalid CSV
+    Given I have a CSV with the following content:
+    """
+	"Foo",	"Bar"	,	"Baz
+    """
+    And it is stored at the url "http://example.com/example1.csv"
+    When I ask if the CSV is valid
+    Then I should get the value of false
+    
+   Scenario: Successfully report a CSV with incorrect quoting
+    Given I have a CSV with the following content:
+    """
+"Foo","Bar","Baz
+    """
+    And it is stored at the url "http://example.com/example1.csv"
+    When I ask if the CSV is valid
+    Then I should get the value of false      
+    
+   Scenario: Successfully report a CSV with incorrect whitespace
+    Given I have a CSV with the following content:
+    """
+"Foo","Bar",   "Baz"
+    """
+    And it is stored at the url "http://example.com/example1.csv"
+    When I ask if the CSV is valid
+    Then I should get the value of false     
+    
+   Scenario: Successfully report a CSV with ragged rows
+    Given I have a CSV with the following content:
+    """
+"col1","col2","col2"
+"1","2","3"
+"4","5"
+    """
+    And it is stored at the url "http://example.com/example1.csv"
+    When I ask if the CSV is valid
+    Then I should get the value of false
+
+    Scenario: Don't class blank values as inconsistencies
+     Given I have a CSV with the following content:
+     """
+"col1","col2","col3"
+"1","2","3"
+"4","5","6"
+"","7","8"
+"9","10","11"
+"","12","13"
+"","14","15"
+"16","17","18"
+     """
+     And it is stored at the url "http://example.com/example1.csv"
+     When I ask if there are warnings
+     Then there should be 0 warnings

data/features/schema_validation.feature

@@ -0,0 +1,63 @@
+Feature: Schema Validation
+
+  Scenario: Valid CSV
+    Given I have a CSV with the following content:
+    """
+"Bob","1234","bob@example.org"
+"Alice","5","alice@example.com"
+    """
+    And it is stored at the url "http://example.com/example1.csv"
+    And I have a schema with the following content:
+    """
+{
+	"fields": [
+          { "name": "Name", "constraints": { "required": true } },
+          { "name": "Id", "constraints": { "required": true, "minLength": 1 } },
+          { "name": "Email", "constraints": { "required": true } }
+    ]
+}
+    """
+    When I ask if there are errors
+    Then there should be 0 error
+
+  Scenario: Schema invalid CSV
+    Given I have a CSV with the following content:
+    """
+"Bob","1234","bob@example.org"
+"Alice","5","alice@example.com"
+    """
+    And it is stored at the url "http://example.com/example1.csv"
+    And I have a schema with the following content:
+    """
+{
+	"fields": [
+          { "name": "Name", "constraints": { "required": true } },
+          { "name": "Id", "constraints": { "required": true, "minLength": 3 } },
+          { "name": "Email", "constraints": { "required": true } }
+    ]
+}
+    """
+    When I ask if there are errors
+    Then there should be 1 error
+
+  Scenario: CSV with incorrect header
+    Given I have a CSV with the following content:
+    """
+"name","id","contact"
+"Bob","1234","bob@example.org"
+"Alice","5","alice@example.com"
+    """
+    And it is stored at the url "http://example.com/example1.csv"
+    And I have a schema with the following content:
+    """
+{
+	"fields": [
+          { "name": "name", "constraints": { "required": true } },
+          { "name": "id", "constraints": { "required": true, "minLength": 3 } },
+          { "name": "email", "constraints": { "required": true } }
+    ]
+}
+    """
+    When I ask if there are warnings
+    Then there should be 1 warnings
+    

data/features/sources.feature

@@ -0,0 +1,18 @@
+Feature: Parse CSV from Different Sources
+  
+  Scenario: Successfully parse a valid CSV from a StringIO
+      Given I have a CSV with the following content:
+    """
+"Foo","Bar","Baz"
+"1","2","3"
+"3","2","1"
+    """
+    And it is parsed as a StringIO
+    When I ask if the CSV is valid
+    Then I should get the value of true
+  
+  Scenario: Successfully parse a valid CSV from a File
+    Given I parse a CSV file called "valid.csv"
+    When I ask if the CSV is valid
+    Then I should get the value of true
+  

data/features/step_definitions/csv_options_steps.rb

@@ -0,0 +1,19 @@
+Given(/^I set the delimiter to "(.*?)"$/) do |delimiter|
+  @csv_options ||= default_csv_options
+  @csv_options["delimiter"] = delimiter
+end
+
+Given(/^I set quotechar to "(.*?)"$/) do |doublequote|
+  @csv_options ||= default_csv_options
+  @csv_options["quoteChar"] = doublequote
+end
+
+Given(/^I set the line endings to linefeed$/) do
+  @csv_options ||= default_csv_options
+  @csv_options["lineTerminator"] = "\n"
+end
+
+Given(/^I set header to "(.*?)"$/) do |boolean|
+  @csv_options ||= default_csv_options
+  @csv_options["header"] = boolean == "true"
+end

data/features/step_definitions/information_steps.rb

@@ -0,0 +1,13 @@
+Given(/^the content type is "(.*?)"$/) do |arg1|
+  @content_type = "text/csv"
+end
+
+Then(/^the "(.*?)" should be "(.*?)"$/) do |type, encoding|
+  validator = Csvlint::Validator.new( @url, default_csv_options ) 
+  validator.send(type.to_sym).should == encoding
+end
+
+Then(/^the metadata content type should be "(.*?)"$/) do |content_type|
+  validator = Csvlint::Validator.new( @url, default_csv_options ) 
+  validator.headers['content-type'].should == content_type
+end

data/features/step_definitions/parse_csv_steps.rb

@@ -0,0 +1,30 @@
+Given(/^I have a CSV with the following content:$/) do |string|
+  @csv = string
+end
+
+Given(/^it is stored at the url "(.*?)"$/) do |url|
+  @url = url
+  content_type = @content_type || "text/csv"
+  charset = @encoding || "UTF-8"
+  stub_request(:get, url).to_return(:status => 200, :body => @csv, :headers => {"Content-Type" => "#{content_type}; charset=#{charset}"})
+end
+
+Given(/^it is stored at the url "(.*?)" with no character set$/) do |url|
+  @url = url
+  content_type = @content_type || "text/csv"
+  stub_request(:get, url).to_return(:status => 200, :body => @csv, :headers => {"Content-Type" => "#{content_type}"})
+end
+
+When(/^I ask if the CSV is valid$/) do
+  @csv_options ||= default_csv_options
+  @validator = Csvlint::Validator.new( @url, @csv_options ) 
+  @valid = @validator.valid?
+end
+
+Then(/^I should get the value of true$/) do
+  expect( @valid ).to be(true)
+end
+
+Then(/^I should get the value of false$/) do
+  expect( @valid ).to be(false)
+end

data/features/step_definitions/schema_validation_steps.rb

@@ -0,0 +1,7 @@
+Given(/^I have a schema with the following content:$/) do |json|
+  @schema_json = json
+end
+
+Given(/^the schema is stored at the url "(.*?)"$/) do |schema_url|
+  @schema_url = schema_url
+end

data/features/step_definitions/sources_steps.rb

@@ -0,0 +1,7 @@
+Given(/^it is parsed as a StringIO$/) do
+  @url = StringIO.new(@csv)
+end
+
+Given(/^I parse a CSV file called "(.*?)"$/) do |filename|
+  @url = File.new( File.join( File.dirname(__FILE__), "..", "fixtures", filename ) )
+end

data/features/step_definitions/validation_errors_steps.rb

@@ -0,0 +1,43 @@
+When(/^I ask if there are errors$/) do
+  @csv_options ||= default_csv_options
+  
+  if @schema_json
+    @schema = Csvlint::Schema.from_json_table( @schema_url || "http://example.org ", JSON.parse(@schema_json) )
+  end
+  
+  @validator = Csvlint::Validator.new( @url, @csv_options, @schema ) 
+  @errors = @validator.errors
+end
+
+Then(/^there should be (\d+) error$/) do |count|  
+  @errors.count.should == count.to_i
+end
+
+Then(/^that error should have the type "(.*?)"$/) do |type|
+  @errors.first.type.should == type.to_sym
+end
+
+Then(/^that error should have the row "(.*?)"$/) do |row|
+  @errors.first.row.should == row.to_i
+end
+
+Then(/^that error should have the column "(.*?)"$/) do |column|
+  @errors.first.column.should == column.to_i
+end
+
+Then(/^that error should have the content "(.*)"$/) do |content|
+  @errors.first.content.chomp.should == content.chomp
+end
+
+Then(/^that error should have no content$/) do
+  @errors.first.content.should == nil
+end
+
+Given(/^I have a CSV that doesn't exist$/) do
+  @url = "http//www.example.com/fake-csv.csv"
+  stub_request(:get, @url).to_return(:status => 404)
+end
+
+Then(/^there should be no "(.*?)" errors$/) do |type|
+  @errors.each do |error| error.type.should_not == type.to_sym end
+end

data/features/step_definitions/validation_info_steps.rb

@@ -0,0 +1,18 @@
+Given(/^I ask if there are info messages$/) do
+  @csv_options ||= default_csv_options
+  
+  if @schema_json
+    @schema = Csvlint::Schema.from_json_table( @schema_url || "http://example.org ", JSON.parse(@schema_json) )
+  end
+  
+  @validator = Csvlint::Validator.new( @url, @csv_options, @schema ) 
+  @info_messages = @validator.info_messages
+end
+
+Then(/^there should be (\d+) info messages?$/) do |num|
+  @info_messages.count.should == num.to_i
+end
+
+Then(/^that message should have the type "(.*?)"$/) do |msg_type|
+  @info_messages.first.type.should == msg_type.to_sym
+end

data/features/step_definitions/validation_warnings_steps.rb

@@ -0,0 +1,46 @@
+Given(/^it is encoded as "(.*?)"$/) do |encoding|
+  @csv = @csv.encode(encoding)
+  @encoding = encoding
+end
+
+Given(/^I set an encoding header of "(.*?)"$/) do |encoding|
+  @encoding = encoding
+end
+
+Given(/^I do not set an encoding header$/) do
+  @encoding = nil
+end
+
+Given(/^I have a CSV file called "(.*?)"$/) do |filename|
+  @csv = File.read( File.join( File.dirname(__FILE__), "..", "fixtures", filename ) )
+end
+
+When(/^I ask if there are warnings$/) do
+  @csv_options ||= default_csv_options
+  if @schema_json
+    @schema = Csvlint::Schema.from_json_table( @schema_url || "http://example.org ", JSON.parse(@schema_json) )
+  end
+
+  @validator = Csvlint::Validator.new( @url, @csv_options, @schema ) 
+  @warnings = @validator.warnings
+end
+
+Then(/^there should be (\d+) warnings$/) do |count|
+  @warnings.count.should == count.to_i
+end
+
+Given(/^the content type is set to "(.*?)"$/) do |type|
+  @content_type = type
+end
+
+Then(/^that warning should have the row "(.*?)"$/) do |row|
+  @warnings.first.row.should == row.to_i
+end
+
+Then(/^that warning should have the column "(.*?)"$/) do |column|
+  @warnings.first.column.should == column.to_i
+end
+
+Then(/^that warning should have the type "(.*?)"$/) do |type|
+  @warnings.first.type.should == type.to_sym
+end