csvlint 0.0.1

data/spec/field_spec.rb

@@ -0,0 +1,247 @@
+require 'spec_helper'
+
+describe Csvlint::Field do
+  
+  it "should validate required fields" do
+    field = Csvlint::Field.new("test", { "required" => true } )
+    expect( field.validate_column( nil ) ).to be(false)  
+    expect( field.errors.first.category ).to be(:schema)
+    expect( field.validate_column( "" ) ).to be(false)
+    expect( field.validate_column( "data" ) ).to be(true)
+  end
+  
+  it "should include the failed constraints" do
+    field = Csvlint::Field.new("test", { "required" => true } )
+    expect( field.validate_column( nil ) ).to be(false)  
+    expect( field.errors.first.constraints ).to eql( { "required" => true } )
+  end
+  
+  it "should validate minimum length" do
+    field = Csvlint::Field.new("test", { "minLength" => 3 } )
+    expect( field.validate_column( nil ) ).to be(false)
+    expect( field.validate_column( "" ) ).to be(false)    
+    expect( field.validate_column( "ab" ) ).to be(false)
+    expect( field.validate_column( "abc" ) ).to be(true)
+    expect( field.validate_column( "abcd" ) ).to be(true)    
+  end
+  
+  it "should validate maximum length" do
+    field = Csvlint::Field.new("test", { "maxLength" => 3 } )
+    expect( field.validate_column( nil ) ).to be(true)
+    expect( field.validate_column( "" ) ).to be(true)    
+    expect( field.validate_column( "ab" ) ).to be(true)
+    expect( field.validate_column( "abc" ) ).to be(true)
+    expect( field.validate_column( "abcd" ) ).to be(false)
+  end
+  
+  it "should validate against regex" do
+    field = Csvlint::Field.new("test", { "pattern" => "\{[a-fA-F0-9]{8}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{12}\}"} )
+    expect( field.validate_column( "abc") ).to be(false)
+    expect( field.validate_column( "{3B0DA29C-C89A-4FAA-918A-0000074FA0E0}") ).to be(true)  
+  end
+  
+  it "should apply combinations of constraints" do
+    field = Csvlint::Field.new("test", { "required"=>true, "pattern" => "\{[a-fA-F0-9]{8}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{12}\}"} )
+    expect( field.validate_column( "abc") ).to be(false)
+    expect( field.errors.first.constraints ).to eql( { "pattern" => "\{[a-fA-F0-9]{8}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{12}\}" } )
+
+    expect( field.validate_column( nil ) ).to be(false)
+    expect( field.errors.first.constraints ).to eql( { "required"=>true } )
+    
+    expect( field.validate_column( "{3B0DA29C-C89A-4FAA-918A-0000074FA0E0}") ).to be(true)  
+    
+  end
+
+  it "should enforce uniqueness for a column" do
+    field = Csvlint::Field.new("test", { "unique" => true } )
+    expect( field.validate_column( "abc") ).to be(true)
+    expect( field.validate_column( "abc") ).to be(false)
+    expect( field.errors.first.category ).to be(:schema)
+    expect( field.errors.first.type ).to be(:unique)
+  end
+
+  context "it should validate correct types" do
+    it "skips empty fields" do
+      field = Csvlint::Field.new("test", { "type" => "http://www.w3.org/2001/XMLSchema#int" })
+      expect( field.validate_column("")).to be(true)
+    end
+    
+    it "validates strings" do
+      field = Csvlint::Field.new("test", { "type" => "http://www.w3.org/2001/XMLSchema#string" })
+      expect( field.validate_column("42")).to be(true)
+      expect( field.validate_column("forty-two")).to be(true)
+    end
+    
+    it "validates ints" do
+      field = Csvlint::Field.new("test", { "type" => "http://www.w3.org/2001/XMLSchema#int" })
+      expect( field.validate_column("42")).to be(true)
+      expect( field.validate_column("forty-two")).to be(false)
+    end
+
+    it "validates integers" do
+      field = Csvlint::Field.new("test", { "type" => "http://www.w3.org/2001/XMLSchema#integer" })
+      expect( field.validate_column("42")).to be(true)
+      expect( field.validate_column("forty-two")).to be(false)
+    end        
+
+    it "validates floats" do
+      field = Csvlint::Field.new("test", { "type" => "http://www.w3.org/2001/XMLSchema#float" })
+      expect(field.validate_column("42.0")).to be(true)
+      expect(field.validate_column("42")).to be(true)
+      expect(field.validate_column("forty-two")).to be(false)
+    end
+
+    it "validates URIs" do
+      field = Csvlint::Field.new("test", { "type" => "http://www.w3.org/2001/XMLSchema#anyURI" })
+      expect(field.validate_column("http://theodi.org/team")).to be(true)
+      expect(field.validate_column("https://theodi.org/team")).to be(true)
+      expect(field.validate_column("42.0")).to be(false)
+    end
+
+    it "validates booleans" do
+      field = Csvlint::Field.new("test", { "type" => "http://www.w3.org/2001/XMLSchema#boolean" })
+      expect(field.validate_column("true")).to be(true)
+      expect(field.validate_column("1")).to be(true)
+      expect(field.validate_column("false")).to be(true)
+      expect(field.validate_column("0")).to be(true)
+      expect(field.validate_column("derp")).to be(false)
+    end
+
+    context "it should validate all kinds of integers" do
+      it "validates a non-positive integer" do
+        field = Csvlint::Field.new("test", { "type" => "http://www.w3.org/2001/XMLSchema#nonPositiveInteger" })
+        expect(field.validate_column("0")).to be(true)
+        expect(field.validate_column("-1")).to be(true)
+        expect(field.validate_column("1")).to be(false)
+      end
+
+      it "validates a negative integer" do
+        field = Csvlint::Field.new("test", { "type" => "http://www.w3.org/2001/XMLSchema#negativeInteger" })
+        expect(field.validate_column("0")).to be(false)
+        expect(field.validate_column("-1")).to be(true)
+        expect(field.validate_column("1")).to be(false)
+      end
+
+      it "validates a non-negative integer" do
+        field = Csvlint::Field.new("test", { "type" => "http://www.w3.org/2001/XMLSchema#nonNegativeInteger" })
+        expect(field.validate_column("0")).to be(true)
+        expect(field.validate_column("-1")).to be(false)
+        expect(field.validate_column("1")).to be(true)
+      end
+
+      it "validates a positive integer" do
+        field = Csvlint::Field.new("test", { "type" => "http://www.w3.org/2001/XMLSchema#positiveInteger" })
+        expect(field.validate_column("0")).to be(false)
+        expect(field.validate_column("-1")).to be(false)
+        expect(field.errors.first.constraints).to eql( { "type" => "http://www.w3.org/2001/XMLSchema#positiveInteger" } )
+        expect(field.validate_column("1")).to be(true)
+      end
+    end
+    
+    context "when validating ranges" do
+      
+      it "should enforce minimum values" do
+        field = Csvlint::Field.new("test", { 
+            "type" => "http://www.w3.org/2001/XMLSchema#int",
+            "minimum" => "40"
+        })
+        expect( field.validate_column("42")).to be(true)
+
+        field = Csvlint::Field.new("test", { 
+            "type" => "http://www.w3.org/2001/XMLSchema#int",
+            "minimum" => "40"
+        })
+        expect( field.validate_column("39")).to be(false)     
+        expect( field.errors.first.type ).to eql(:below_minimum)
+      end
+      
+      it "should enforce maximum values" do
+        field = Csvlint::Field.new("test", { 
+            "type" => "http://www.w3.org/2001/XMLSchema#int",
+            "maximum" => "40"
+        })
+        expect( field.validate_column("39")).to be(true)
+
+        field = Csvlint::Field.new("test", { 
+            "type" => "http://www.w3.org/2001/XMLSchema#int",
+            "maximum" => "40"
+        })
+        expect( field.validate_column("41")).to be(false)        
+        expect( field.errors.first.type ).to eql(:above_maximum)
+
+      end      
+    end
+    
+    context "when validating dates" do
+      it "should validate a date time" do
+        field = Csvlint::Field.new("test", { 
+             "type" => "http://www.w3.org/2001/XMLSchema#dateTime"
+         })
+         expect( field.validate_column("2014-02-17T11:09:00Z")).to be(true)
+         expect( field.validate_column("invalid-date")).to be(false)
+        expect( field.validate_column("2014-02-17")).to be(false)          
+      end
+      it "should validate a date" do
+        field = Csvlint::Field.new("test", { 
+             "type" => "http://www.w3.org/2001/XMLSchema#date"
+         })
+         expect( field.validate_column("2014-02-17T11:09:00Z")).to be(false)
+         expect( field.validate_column("invalid-date")).to be(false)
+        expect( field.validate_column("2014-02-17")).to be(true)          
+      end
+      it "should validate a time" do
+        field = Csvlint::Field.new("test", { 
+             "type" => "http://www.w3.org/2001/XMLSchema#time"
+         })
+         expect( field.validate_column("11:09:00")).to be(true)
+         expect( field.validate_column("2014-02-17T11:09:00Z")).to be(false)
+         expect( field.validate_column("not-a-time")).to be(false)     
+         expect( field.validate_column("27:97:00")).to be(false)   
+      end
+      it "should validate a year" do
+        field = Csvlint::Field.new("test", { 
+             "type" => "http://www.w3.org/2001/XMLSchema#gYear"
+         })
+         expect( field.validate_column("1999")).to be(true)
+         expect( field.validate_column("2525")).to be(true)
+         expect( field.validate_column("0001")).to be(true)
+         expect( field.validate_column("2014-02-17T11:09:00Z")).to be(false)
+         expect( field.validate_column("not-a-time")).to be(false)     
+         expect( field.validate_column("27:97:00")).to be(false)   
+      end
+      it "should validate a year-month" do
+        field = Csvlint::Field.new("test", { 
+             "type" => "http://www.w3.org/2001/XMLSchema#gYearMonth"
+         })
+         expect( field.validate_column("1999-12")).to be(true)
+         expect( field.validate_column("2525-01")).to be(true)
+         expect( field.validate_column("2014-02-17T11:09:00Z")).to be(false)
+         expect( field.validate_column("not-a-time")).to be(false)     
+         expect( field.validate_column("27:97:00")).to be(false)   
+      end
+      it "should allow user to specify custom date time pattern" do
+        field = Csvlint::Field.new("test", { 
+             "type" => "http://www.w3.org/2001/XMLSchema#dateTime",
+             "datePattern" => "%Y-%m-%d %H:%M:%S"
+         })
+         expect( field.validate_column("1999-12-01 10:00:00")).to be(true)
+         expect( field.validate_column("invalid-date")).to be(false)
+        expect( field.validate_column("2014-02-17")).to be(false)  
+        expect( field.errors.first.constraints ).to eql( { 
+             "type" => "http://www.w3.org/2001/XMLSchema#dateTime",
+             "datePattern" => "%Y-%m-%d %H:%M:%S"
+         })
+         
+      end
+      it "should allow user to compare dates" do
+        field = Csvlint::Field.new("test", { 
+             "type" => "http://www.w3.org/2001/XMLSchema#dateTime",
+             "datePattern" => "%Y-%m-%d %H:%M:%S",
+             "minimum" => "1990-01-01 10:00:00"
+         })
+         expect( field.validate_column("1999-12-01 10:00:00")).to be(true)
+         expect( field.validate_column("1989-12-01 10:00:00")).to be(false)        
+      end
+    end
+  end
+end

data/spec/schema_spec.rb

@@ -0,0 +1,149 @@
+require 'spec_helper'
+
+describe Csvlint::Schema do
+  
+  it "should tolerate missing fields" do
+    schema = Csvlint::Schema.from_json_table("http://example.org", {})
+    expect( schema ).to_not be(nil)
+    expect( schema.fields.empty? ).to eql(true)
+  end
+
+  it "should tolerate fields with no constraints" do
+    schema = Csvlint::Schema.from_json_table("http://example.org", {
+      "fields" => [ { "name" => "test" } ]
+    })
+    expect( schema ).to_not be(nil)
+    expect( schema.fields[0].name ).to eql("test")
+    expect( schema.fields[0].constraints ).to eql({})
+  end
+      
+  it "should validate against the schema" do
+    field = Csvlint::Field.new("test", { "required" => true } )
+    field2 = Csvlint::Field.new("test", { "minLength" => 3 } )
+    schema = Csvlint::Schema.new("http://example.org", [field, field2] )
+    
+    expect( schema.validate_row( ["", "x"] ) ).to eql(false)
+    expect( schema.errors.size ).to eql(2)
+    expect( schema.errors.first.type).to eql(:missing_value)
+    expect( schema.errors.first.category).to eql(:schema)
+    expect( schema.errors.first.column).to eql(1)
+    expect( schema.validate_row( ["abc", "1234"] ) ).to eql(true)
+      
+  end
+  
+  it "should include validations for missing columns" do
+    minimum = Csvlint::Field.new("test", { "minLength" => 3 } )
+    required = Csvlint::Field.new("test2", { "required" => true } )
+    schema = Csvlint::Schema.new("http://example.org", [minimum, required] )
+    
+    expect( schema.validate_row( ["abc", "x"] ) ).to eql(true)
+      
+    expect( schema.validate_row( ["abc"] ) ).to eql(false)
+    expect( schema.errors.size ).to eql(1)
+    expect( schema.errors.first.type).to eql(:missing_value)
+
+    schema = Csvlint::Schema.new("http://example.org", [required, minimum] )
+    expect( schema.validate_row( ["abc"] ) ).to eql(false)
+    expect( schema.errors.size ).to eql(1)
+    expect( schema.errors.first.type).to eql(:min_length)    
+  end
+  
+  it "should warn if the data has fewer columns" do
+    minimum = Csvlint::Field.new("test", { "minLength" => 3 } )
+    required = Csvlint::Field.new("test2", { "maxLength" => 5 } )
+    schema = Csvlint::Schema.new("http://example.org", [minimum, required] )
+
+    expect( schema.validate_row( ["abc"], 1 ) ).to eql(true)
+    expect( schema.warnings.size ).to eql(1)
+    expect( schema.warnings.first.type).to eql(:missing_column)
+    expect( schema.warnings.first.category).to eql(:schema)
+    expect( schema.warnings.first.row).to eql(1)
+    expect( schema.warnings.first.column).to eql(2)
+
+    #no ragged row error    
+    expect( schema.errors.size ).to eql(0)
+  end
+  
+  it "should warn if the data has additional columns" do
+    minimum = Csvlint::Field.new("test", { "minLength" => 3 } )
+    required = Csvlint::Field.new("test2", { "required" => true } )
+    schema = Csvlint::Schema.new("http://example.org", [minimum, required] )
+
+    expect( schema.validate_row( ["abc", "x", "more", "columns"], 1 ) ).to eql(true)
+    expect( schema.warnings.size ).to eql(2)
+    expect( schema.warnings.first.type).to eql(:extra_column)
+    expect( schema.warnings.first.category).to eql(:schema)
+    expect( schema.warnings.first.row).to eql(1)
+    expect( schema.warnings.first.column).to eql(3)
+
+    expect( schema.warnings[1].type).to eql(:extra_column)
+    expect( schema.warnings[1].column).to eql(4)
+
+    #no ragged row error    
+    expect( schema.errors.size ).to eql(0)        
+  end  
+
+  context "when validating header" do
+    it "should warn if column names are different to field names" do
+      minimum = Csvlint::Field.new("minimum", { "minLength" => 3 } )
+      required = Csvlint::Field.new("required", { "required" => true } )
+      schema = Csvlint::Schema.new("http://example.org", [minimum, required] )
+  
+      expect( schema.validate_header(["minimum", "required"]) ).to eql(true)
+      expect( schema.warnings.size ).to eql(0)
+      
+      expect( schema.validate_header(["wrong", "required"]) ).to eql(true)
+      expect( schema.warnings.size ).to eql(1)
+      expect( schema.warnings.first.type).to eql(:header_name)
+      expect( schema.warnings.first.content).to eql("wrong")
+      expect( schema.warnings.first.column).to eql(1)
+      expect( schema.warnings.first.category).to eql(:schema)
+      
+      expect( schema.validate_header(["minimum", "Required"]) ).to eql(true)
+      expect( schema.warnings.size ).to eql(1)
+
+    end        
+  end  
+  
+  context "when parsing JSON Tables" do
+    
+    before(:each) do 
+      @example=<<-EOL
+      {
+          "title": "Schema title",
+          "description": "schema", 
+          "fields": [
+              { "name": "ID", "constraints": { "required": true }, "title": "id", "description": "house identifier" },
+              { "name": "Price", "constraints": { "required": true, "minLength": 1 } },
+              { "name": "Postcode", "constraints": { "required": true, "pattern": "[A-Z]{1,2}[0-9][0-9A-Z]? ?[0-9][A-Z]{2}" } }
+          ]
+      }
+  EOL
+      stub_request(:get, "http://example.com/example.json").to_return(:status => 200, :body => @example)
+    end
+    
+    it "should create a schema from a pre-parsed JSON table" do
+      json = JSON.parse( @example )
+      schema = Csvlint::Schema.from_json_table("http://example.org", json)
+      
+      expect( schema.uri ).to eql("http://example.org")
+      expect( schema.title ).to eql("Schema title")
+      expect( schema.description ).to eql("schema")
+      expect( schema.fields.length ).to eql(3)
+      expect( schema.fields[0].name ).to eql("ID")
+      expect( schema.fields[0].constraints["required"] ).to eql(true)
+      expect( schema.fields[0].title ).to eql("id")
+      expect( schema.fields[0].description ).to eql("house identifier")
+    end
+    
+    it "should create a schema from a JSON Table URL" do
+      schema = Csvlint::Schema.load_from_json_table("http://example.com/example.json")
+      expect( schema.uri ).to eql("http://example.com/example.json")
+      expect( schema.fields.length ).to eql(3)
+      expect( schema.fields[0].name ).to eql("ID")
+      expect( schema.fields[0].constraints["required"] ).to eql(true)
+      
+    end
+  end  
+  
+end

data/spec/spec_helper.rb

@@ -0,0 +1,20 @@
+require 'simplecov'
+require 'simplecov-rcov'
+require 'csvlint'
+require 'pry'
+require 'webmock/rspec'
+require 'coveralls'
+
+Coveralls.wear_merged!
+
+RSpec.configure do |config|
+  config.treat_symbols_as_metadata_keys_with_true_values = true
+  config.run_all_when_everything_filtered = true
+  config.filter_run :focus
+
+  # Run specs in random order to surface order dependencies. If you find an
+  # order dependency and want to debug it, you can fix the order by providing
+  # the seed, which is printed after each run.
+  #     --seed 1234
+  config.order = 'random'
+end

data/spec/validator_spec.rb

@@ -0,0 +1,279 @@
+require 'spec_helper'
+
+describe Csvlint::Validator do
+    
+  before do
+    stub_request(:get, "http://example.com/example.csv").to_return(:status => 200, :body => "")
+  end
+  
+  context "csv dialect" do
+    it "should provide sensible defaults for CSV parsing" do
+      validator = Csvlint::Validator.new("http://example.com/example.csv")
+      opts = validator.instance_variable_get("@csv_options")
+      opts.should include({
+        :col_sep => ",",
+        :row_sep => :auto,
+        :quote_char => '"',
+        :skip_blanks => false
+      })
+    end
+    
+    it "should map CSV DDF to correct values" do
+      validator = Csvlint::Validator.new("http://example.com/example.csv")
+      opts = validator.dialect_to_csv_options( {
+        "lineTerminator" => "\n",
+        "delimiter" => "\t",
+        "quoteChar" => "'"
+      })
+      opts.should include({
+        :col_sep => "\t",
+        :row_sep => "\n",
+        :quote_char => "'",
+        :skip_blanks => false
+      })       
+    end
+    
+  end
+  
+  context "when detecting headers" do
+    it "should default to expecting a header" do
+      validator = Csvlint::Validator.new("http://example.com/example.csv")
+      expect( validator.header? ).to eql(true)
+    end
+    
+    it "should look in CSV options to detect header" do
+      opts = {
+        "header" => true
+      }
+      validator = Csvlint::Validator.new("http://example.com/example.csv", opts)
+      expect( validator.header? ).to eql(true)
+      opts = {
+        "header" => false
+      }
+      validator = Csvlint::Validator.new("http://example.com/example.csv", opts)
+      expect( validator.header? ).to eql(false)      
+    end
+    
+    it "should look in content-type for header" do
+      stub_request(:get, "http://example.com/example.csv").to_return(:status => 200, :headers=>{"Content-Type" => "text/csv; header=absent"}, :body => File.read(File.join(File.dirname(__FILE__),'..','features','fixtures','valid.csv')))
+      validator = Csvlint::Validator.new("http://example.com/example.csv")
+      expect( validator.header? ).to eql(false)
+
+      stub_request(:get, "http://example.com/example.csv").to_return(:status => 200, :headers=>{"Content-Type" => "text/csv; header=present"}, :body => File.read(File.join(File.dirname(__FILE__),'..','features','fixtures','valid.csv')))
+      validator = Csvlint::Validator.new("http://example.com/example.csv")
+      expect( validator.header? ).to eql(true)              
+    end  
+    
+  end
+  
+  context "when validating headers" do
+    it "should warn if column names aren't unique" do      
+      data = StringIO.new( "minimum, minimum" )
+      validator = Csvlint::Validator.new(data)
+      expect( validator.validate_header(["minimum", "minimum"]) ).to eql(true)
+      expect( validator.warnings.size ).to eql(1)
+      expect( validator.warnings.first.type).to eql(:duplicate_column_name)
+      expect( validator.warnings.first.category).to eql(:schema)
+    end
+
+    it "should warn if column names are blank" do
+      data = StringIO.new( "minimum," )
+      validator = Csvlint::Validator.new(data)
+      
+      expect( validator.validate_header(["minimum", ""]) ).to eql(true)
+      expect( validator.warnings.size ).to eql(1)
+      expect( validator.warnings.first.type).to eql(:empty_column_name)
+      expect( validator.warnings.first.category).to eql(:schema)
+    end
+    
+    it "should include info message about missing header when we have assumed a header" do
+      data = StringIO.new( "1,2,3\r\n" )
+      validator = Csvlint::Validator.new(data)
+      
+      expect( validator.valid? ).to eql(true)
+      expect( validator.info_messages.size ).to eql(1)
+      expect( validator.info_messages.first.type).to eql(:assumed_header)
+      expect( validator.info_messages.first.category).to eql(:structure)
+    end  
+
+    it "should not include info message about missing header when we are told about the header" do
+      data = StringIO.new( "1,2,3\r\n" )
+      validator = Csvlint::Validator.new(data, "header"=>false)
+      
+      expect( validator.valid? ).to eql(true)
+      expect( validator.info_messages.size ).to eql(0)
+    end
+    
+    it "should be an error if we have assumed a header, there is no dialect and there's no content-type" do
+      stub_request(:get, "http://example.com/example.csv").to_return(:status => 200, :headers=>{}, :body => File.read(File.join(File.dirname(__FILE__),'..','features','fixtures','valid.csv')))
+      validator = Csvlint::Validator.new("http://example.com/example.csv")
+      expect( validator.valid? ).to eql(false)   
+    end  
+
+    it "should be an error if we have assumed a header, there is no dialect and content-type doesn't declare header" do
+      stub_request(:get, "http://example.com/example.csv").to_return(:status => 200, :headers=>{"Content-Type" => "text/csv"}, :body => File.read(File.join(File.dirname(__FILE__),'..','features','fixtures','valid.csv')))
+      validator = Csvlint::Validator.new("http://example.com/example.csv")
+      expect( validator.valid? ).to eql(false)   
+    end  
+        
+    it "should be valid if we have a dialect and the data is from the web" do
+      stub_request(:get, "http://example.com/example.csv").to_return(:status => 200, :headers=>{"Content-Type" => "text/csv"}, :body => File.read(File.join(File.dirname(__FILE__),'..','features','fixtures','valid.csv')))
+      #header defaults to true in csv dialect, so this is valid
+      validator = Csvlint::Validator.new("http://example.com/example.csv", {})
+      expect( validator.valid? ).to eql(true)   
+
+      stub_request(:get, "http://example.com/example.csv").to_return(:status => 200, :headers=>{"Content-Type" => "text/csv"}, :body => File.read(File.join(File.dirname(__FILE__),'..','features','fixtures','valid.csv')))      
+      validator = Csvlint::Validator.new("http://example.com/example.csv", {"header"=>true})
+      expect( validator.valid? ).to eql(true) 
+      
+      stub_request(:get, "http://example.com/example.csv").to_return(:status => 200, :headers=>{"Content-Type" => "text/csv"}, :body => File.read(File.join(File.dirname(__FILE__),'..','features','fixtures','valid.csv')))      
+      validator = Csvlint::Validator.new("http://example.com/example.csv", {"header"=>false})
+      expect( validator.valid? ).to eql(true) 
+    end
+                        
+  end
+  
+  context "build_formats" do
+    
+    {
+      "string" => "foo",
+      "numeric" => "1",
+      "uri" => "http://www.example.com",
+      "dateTime_iso8601" => "2013-01-01T13:00:00Z",
+      "date_db" => "2013-01-01",
+      "dateTime_hms" => "13:00:00"
+    }.each do |type, content|
+      it "should return the format of #{type} correctly" do
+        row = [content]
+        
+        validator = Csvlint::Validator.new("http://example.com/example.csv")
+        validator.build_formats(row, 1)
+        formats = validator.instance_variable_get("@formats")
+        
+        formats[0].first.should == type
+      end
+    end
+    
+    it "treats floats and ints the same" do
+      row = ["12", "3.1476"]
+      
+      validator = Csvlint::Validator.new("http://example.com/example.csv")
+      validator.build_formats(row, 1)
+      formats = validator.instance_variable_get("@formats")
+      
+      formats[0].first.should == "numeric"
+      formats[1].first.should == "numeric"
+    end
+  
+    it "should ignore blank arrays" do
+      row = []
+    
+      validator = Csvlint::Validator.new("http://example.com/example.csv")
+      validator.build_formats(row, 1)
+      formats = validator.instance_variable_get("@formats") 
+      formats.should == []
+    end
+    
+    it "should work correctly for single columns" do
+      rows = [
+          ["foo"],
+          ["bar"],
+          ["baz"]
+        ]
+        
+      validator = Csvlint::Validator.new("http://example.com/example.csv")
+      
+      rows.each_with_index do |row, i|
+        validator.build_formats(row, i)
+      end
+      
+      formats = validator.instance_variable_get("@formats")
+      
+      formats.should == [
+          ["string",
+          "string",
+          "string"]
+        ]
+    end
+  
+    it "should return formats correctly if a row is blank" do
+      rows = [
+          [],
+          ["foo","1","$2345"]
+        ]
+      
+      validator = Csvlint::Validator.new("http://example.com/example.csv")
+      
+      rows.each_with_index do |row, i|
+        validator.build_formats(row, i)
+      end
+      
+      formats = validator.instance_variable_get("@formats") 
+            
+      formats.should == [
+        ["string"],
+        ["numeric"],
+        ["string"]
+      ]
+    end
+    
+  end
+  
+  context "check_consistency" do
+    
+    it "should return a warning if columns have inconsistent values" do
+      formats = [
+          ["string", "string", "string"],
+          ["string", "numeric", "string"],
+          ["numeric", "numeric", "numeric"],
+        ]
+        
+      validator = Csvlint::Validator.new("http://example.com/example.csv")
+      validator.instance_variable_set("@formats", formats)
+      validator.check_consistency
+          
+      warnings = validator.instance_variable_get("@warnings") 
+      warnings.delete_if { |h| h.type != :inconsistent_values }
+      
+      warnings.count.should == 1
+    end
+    
+  end
+  
+  context "accessing metadata" do
+   
+    before :all do
+      stub_request(:get, "http://example.com/crlf.csv").to_return(:status => 200, :body => File.read(File.join(File.dirname(__FILE__),'..','features','fixtures','windows-line-endings.csv')))
+    end
+    
+    it "can get line break symbol" do
+
+      validator = Csvlint::Validator.new("http://example.com/crlf.csv")
+      validator.line_breaks.should == "\r\n"
+      
+    end
+    
+  end
+  
+  it "should give access to the complete CSV data file" do
+    stub_request(:get, "http://example.com/example.csv").to_return(:status => 200, 
+        :headers=>{"Content-Type" => "text/csv; header=present"}, 
+        :body => File.read(File.join(File.dirname(__FILE__),'..','features','fixtures','valid.csv')))
+    validator = Csvlint::Validator.new("http://example.com/example.csv")    
+    expect( validator.valid? ).to eql(true)
+    data = validator.data
+    expect( data.count ).to eql 4
+    expect( data[0] ).to eql ['Foo','Bar','Baz']
+    expect( data[2] ).to eql ['3','2','1']
+  end
+  
+  it "should follow redirects to SSL" do
+    stub_request(:get, "http://example.com/redirect").to_return(:status => 301, :headers=>{"Location" => "https://example.com/example.csv"})
+    stub_request(:get, "https://example.com/example.csv").to_return(:status => 200, 
+        :headers=>{"Content-Type" => "text/csv; header=present"}, 
+        :body => File.read(File.join(File.dirname(__FILE__),'..','features','fixtures','valid.csv')))
+
+    validator = Csvlint::Validator.new("http://example.com/redirect")    
+    expect( validator.valid? ).to eql(true)
+  end
+end