csvlint 0.0.1

data/lib/csvlint/schema.rb

@@ -0,0 +1,69 @@
+require "set"
+
+module Csvlint
+  
+  class Schema
+    
+    include Csvlint::ErrorCollector
+    
+    attr_reader :uri, :fields, :title, :description
+    
+    def initialize(uri, fields=[], title=nil, description=nil)
+      @uri = uri
+      @fields = fields
+      @title = title
+      @description = description
+      reset
+    end
+
+    def validate_header(header)
+      reset
+      header.each_with_index do |name,i|
+        build_warnings(:header_name, :schema, nil, i+1, name) if fields[i].name != name
+      end
+      return valid?
+    end
+        
+    def validate_row(values, row=nil)
+      reset
+      if values.length < fields.length
+        fields[values.size..-1].each_with_index do |field, i|
+          build_warnings(:missing_column, :schema, row, values.size+i+1)
+        end
+      end
+      if values.length > fields.length
+        values[fields.size..-1].each_with_index do |data_column, i|
+          build_warnings(:extra_column, :schema, row, fields.size+i+1)
+        end
+      end
+      
+      fields.each_with_index do |field,i|
+        value = values[i] || ""
+        result = field.validate_column(value, row, i+1)
+        @errors += fields[i].errors
+        @warnings += fields[i].warnings        
+      end
+            
+      return valid?
+    end
+    
+    def Schema.from_json_table(uri, json)
+      fields = []
+      json["fields"].each do |field_desc|
+        fields << Csvlint::Field.new( field_desc["name"] , field_desc["constraints"], 
+          field_desc["title"], field_desc["description"] )
+      end if json["fields"]
+      return Schema.new( uri , fields, json["title"], json["description"] )
+    end
+    
+    def Schema.load_from_json_table(uri)
+      begin
+        json = JSON.parse( open(uri).read )
+        return Schema.from_json_table(uri,json)
+      rescue
+        return nil
+      end
+    end
+    
+  end
+end

data/lib/csvlint/types.rb

@@ -0,0 +1,113 @@
+require 'set'
+require 'date'
+require 'active_support/core_ext/date/conversions'
+require 'active_support/core_ext/time/conversions'
+
+module Csvlint
+  
+  module Types
+    
+    SIMPLE_FORMATS = {
+      'string'  => lambda { |value, constraints| value },
+      'numeric'     => lambda do |value, constraints| 
+        begin
+          Integer value 
+        rescue ArgumentError
+          Float value
+        end
+      end,
+      'uri'  => lambda do |value, constraints|
+        u = URI.parse value
+        raise ArgumentError unless u.kind_of?(URI::HTTP) || u.kind_of?(URI::HTTPS)
+        u
+      end
+    }
+    
+    def self.date_format(klass = DateTime, value, type)
+      date = klass.strptime(value, klass::DATE_FORMATS[type])
+      raise ArgumentError unless date.to_formatted_s(type) == value
+    end
+    
+    def self.included(base)
+      Time::DATE_FORMATS[:iso8601] = "%Y-%m-%dT%H:%M:%SZ"
+      Time::DATE_FORMATS[:hms] = "%H:%M:%S"
+      
+      Date::DATE_FORMATS.each do |type|
+        SIMPLE_FORMATS["date_#{type.first}"] = lambda do |value, constraints|
+          date_format(Date, value, type.first)
+        end
+      end
+    
+      Time::DATE_FORMATS.each do |type|
+        SIMPLE_FORMATS["dateTime_#{type.first}"] = lambda do |value, constraints|
+          date_format(Time, value, type.first)
+        end
+      end
+    end
+        
+    TYPE_VALIDATIONS = {
+        'http://www.w3.org/2001/XMLSchema#string'  => SIMPLE_FORMATS['string'],
+        'http://www.w3.org/2001/XMLSchema#int'     => lambda { |value, constraints| Integer value },
+        'http://www.w3.org/2001/XMLSchema#integer' => lambda { |value, constraints| Integer value },
+        'http://www.w3.org/2001/XMLSchema#float'   => lambda { |value, constraints| Float value },
+        'http://www.w3.org/2001/XMLSchema#double'   => lambda { |value, constraints| Float value },
+        'http://www.w3.org/2001/XMLSchema#anyURI'  => SIMPLE_FORMATS['uri'],
+        'http://www.w3.org/2001/XMLSchema#boolean' => lambda do |value, constraints|
+          return true if ['true', '1'].include? value
+          return false if ['false', '0'].include? value
+          raise ArgumentError
+        end,
+        'http://www.w3.org/2001/XMLSchema#nonPositiveInteger' => lambda do |value, constraints|
+          i = Integer value
+          raise ArgumentError unless i <= 0
+          i
+        end,
+        'http://www.w3.org/2001/XMLSchema#negativeInteger' => lambda do |value, constraints|
+          i = Integer value
+          raise ArgumentError unless i < 0
+          i
+        end,
+        'http://www.w3.org/2001/XMLSchema#nonNegativeInteger' => lambda do |value, constraints|
+          i = Integer value
+          raise ArgumentError unless i >= 0
+          i
+        end,
+        'http://www.w3.org/2001/XMLSchema#positiveInteger' => lambda do |value, constraints|
+          i = Integer value
+          raise ArgumentError unless i > 0
+          i
+        end,
+        'http://www.w3.org/2001/XMLSchema#dateTime' => lambda do |value, constraints|
+          date_pattern = constraints["datePattern"] || "%Y-%m-%dT%H:%M:%SZ"
+          d = DateTime.strptime(value, date_pattern)
+          raise ArgumentError unless d.strftime(date_pattern) == value
+          d
+        end,
+        'http://www.w3.org/2001/XMLSchema#date' => lambda do |value, constraints|
+          date_pattern = constraints["datePattern"] || "%Y-%m-%d"
+          d = Date.strptime(value, date_pattern)
+          raise ArgumentError unless d.strftime(date_pattern) == value
+          d
+        end,
+        'http://www.w3.org/2001/XMLSchema#time' => lambda do |value, constraints|
+          date_pattern = constraints["datePattern"] || "%H:%M:%S"
+          d = DateTime.strptime(value, date_pattern)
+          raise ArgumentError unless d.strftime(date_pattern) == value
+          d
+        end,
+        'http://www.w3.org/2001/XMLSchema#gYear' => lambda do |value, constraints|
+          date_pattern = constraints["datePattern"] || "%Y"
+          d = Date.strptime(value, date_pattern)
+          raise ArgumentError unless d.strftime(date_pattern) == value
+          d
+        end,     
+        'http://www.w3.org/2001/XMLSchema#gYearMonth' => lambda do |value, constraints|
+          date_pattern = constraints["datePattern"] || "%Y-%m"
+          d = Date.strptime(value, date_pattern)
+          raise ArgumentError unless d.strftime(date_pattern) == value
+          d
+        end 
+    }
+  end
+
+end

data/lib/csvlint/validate.rb

@@ -0,0 +1,253 @@
+require "open_uri_redirections"
+
+module Csvlint
+  
+  class Validator
+    
+    include Csvlint::ErrorCollector
+    include Csvlint::Types
+    
+    attr_reader :encoding, :content_type, :extension, :headers, :line_breaks, :dialect, :csv_header, :schema, :data
+    
+    ERROR_MATCHERS = {
+      "Missing or stray quote" => :stray_quote,
+      "Illegal quoting" => :whitespace,
+      "Unclosed quoted field" => :unclosed_quote,
+    }
+       
+    def initialize(source, dialect = nil, schema = nil)      
+      @source = source
+      @formats = []
+      @schema = schema
+      
+      @assumed_header = true
+      @supplied_dialect = dialect != nil
+      
+      if dialect
+        @assumed_header = false
+      end
+      
+      @dialect = dialect_defaults = {
+        "header" => true,
+        "delimiter" => ",",
+        "skipInitialSpace" => true,
+        "lineTerminator" => :auto,
+        "quoteChar" => '"'
+      }.merge(dialect || {})
+            
+      @csv_header = @dialect["header"]
+        
+      @csv_options = dialect_to_csv_options(@dialect)
+      @extension = parse_extension(source)
+      reset
+      validate
+    end
+        
+    def validate
+      single_col = false   
+      io = nil   
+      begin
+        io = @source.respond_to?(:gets) ? @source : open(@source, :allow_redirections=>:all)
+        validate_metadata(io)
+        parse_csv(io)
+        unless @col_counts.inject(:+).nil?
+          build_warnings(:title_row, :structure) if @col_counts.first < (@col_counts.inject(:+) / @col_counts.count)
+        end
+        build_warnings(:check_options, :structure) if @expected_columns == 1        
+        check_consistency      
+      rescue OpenURI::HTTPError, Errno::ENOENT
+        build_errors(:not_found)
+      ensure
+        io.close if io && io.respond_to?(:close)
+      end
+    end
+    
+    def validate_metadata(io)
+      @encoding = io.charset rescue nil
+      @content_type = io.content_type rescue nil
+      @headers = io.meta rescue nil    
+      if @headers
+        if @headers["content-type"] =~ /header=(present|absent)/
+          @csv_header = true if $1 == "present"
+          @csv_header = false if $1 == "absent"
+          @assumed_header = false
+        end
+        if @headers["content-type"] !~ /charset=/
+          build_warnings(:no_encoding, :context) 
+        else
+          build_warnings(:encoding, :context) if @encoding != "utf-8"
+        end
+        build_warnings(:no_content_type, :context) if @content_type == nil
+        build_warnings(:excel, :context) if @content_type == nil && @extension =~ /.xls(x)?/
+        build_errors(:wrong_content_type, :context) unless (@content_type && @content_type =~ /text\/csv/)
+        
+        if @assumed_header && !@supplied_dialect && (@content_type == nil || @headers["content-type"] !~ /header=(present|absent)/ )
+          build_errors(:undeclared_header, :structure)
+        end
+        
+      end
+      build_info_messages(:assumed_header, :structure) if @assumed_header
+    end
+    
+    def parse_csv(io)
+      @expected_columns = 0
+      current_line = 0
+      reported_invalid_encoding = false
+      @col_counts = []
+      
+      @csv_options[:encoding] = @encoding  
+  
+      begin
+        wrapper = WrappedIO.new( io )
+        csv = CSV.new( wrapper, @csv_options )
+        @data = []
+        @line_breaks = csv.row_sep
+        if @line_breaks != "\r\n"
+          build_info_messages(:nonrfc_line_breaks, :structure)
+        end
+        row = nil
+        loop do
+         current_line = current_line + 1
+         begin
+           row = csv.shift
+           @data << row
+           wrapper.finished
+           if row             
+             if header? && current_line == 1
+               row = row.reject {|r| r.blank? }
+               validate_header(row)
+               @col_counts << row.count
+             else               
+               build_formats(row, current_line)
+               @col_counts << row.reject {|r| r.blank? }.count
+               @expected_columns = row.count unless @expected_columns != 0
+               
+               build_errors(:blank_rows, :structure, current_line, nil, wrapper.line) if row.reject{ |c| c.nil? || c.empty? }.count == 0
+               
+               if @schema
+                 @schema.validate_row(row, current_line)
+                 @errors += @schema.errors
+                 @warnings += @schema.warnings
+               else
+                 build_errors(:ragged_rows, :structure, current_line, nil, wrapper.line) if !row.empty? && row.count != @expected_columns
+               end
+               
+             end
+           else             
+             break
+           end         
+         rescue CSV::MalformedCSVError => e
+           wrapper.finished
+           type = fetch_error(e)
+           if type == :stray_quote && !wrapper.line.match(csv.row_sep)
+             build_errors(:line_breaks, :structure)
+           else
+             build_errors(type, :structure, current_line, nil, wrapper.line)
+           end
+         end
+      end
+      rescue ArgumentError => ae
+        wrapper.finished           
+        build_errors(:invalid_encoding, :structure, current_line, wrapper.line) unless reported_invalid_encoding
+        reported_invalid_encoding = true
+      end
+    end          
+    
+    def validate_header(header)
+      names = Set.new
+      header.each_with_index do |name,i|
+        build_warnings(:empty_column_name, :schema, nil, i+1) if name == ""
+        if names.include?(name)
+          build_warnings(:duplicate_column_name, :schema, nil, i+1)
+        else
+          names << name
+        end
+      end
+      if @schema
+        @schema.validate_header(header)
+        @errors += @schema.errors
+        @warnings += @schema.warnings
+      end
+      return valid?
+    end
+    
+    def header?
+      return @csv_header
+    end
+    
+    def fetch_error(error)
+      e = error.message.match(/^([a-z ]+) (i|o)n line ([0-9]+)\.?$/i)
+      message = e[1] rescue nil
+      ERROR_MATCHERS.fetch(message, :unknown_error)
+    end
+    
+    def dialect_to_csv_options(dialect) 
+        skipinitialspace = dialect["skipInitialSpace"] || true
+        delimiter = dialect["delimiter"]
+        delimiter = delimiter + " " if !skipinitialspace
+        return {
+            :col_sep => delimiter,
+            :row_sep => dialect["lineTerminator"],
+            :quote_char => dialect["quoteChar"],
+            :skip_blanks => false
+        }
+    end
+    
+    def build_formats(row, line) 
+      row.each_with_index do |col, i|
+        next if col.blank?
+        @formats[i] ||= []
+        
+        SIMPLE_FORMATS.each do |type, lambda|
+          begin
+            lambda.call(col, {})
+            @format = type
+          rescue => e
+            nil
+          end
+        end
+        
+        @formats[i] << @format
+      end
+    end
+    
+    def check_consistency
+      percentages = []
+                
+      formats = SIMPLE_FORMATS.map {|type, lambda| type }
+            
+      formats.each do |type, regex|
+        @formats.count.times do |i|
+          percentages[i] ||= {}
+          unless @formats[i].nil?
+            percentages[i][type] = @formats[i].grep(/^#{type}$/).count.to_f / @formats[i].count.to_f
+          end
+        end
+      end
+            
+      percentages.each_with_index do |col, i|
+        next if col.values.blank?
+        build_warnings(:inconsistent_values, :schema, nil, i+1) if col.values.max < 0.9
+      end
+    end
+    
+    private
+    
+    def parse_extension(source)
+      case source
+      when File
+        return File.extname( source.path )
+      when IO
+        return ""
+      when StringIO
+        return ""
+      when Tempfile
+        return ""
+      else
+        parsed = URI.parse(source)
+        File.extname(parsed.path)
+      end
+    end
+    
+  end
+end

data/lib/csvlint/version.rb

@@ -0,0 +1,3 @@
+module Csvlint
+  VERSION = "0.0.1"
+end

data/lib/csvlint/wrapped_io.rb

@@ -0,0 +1,39 @@
+module Csvlint
+  class WrappedIO
+    def initialize(io)
+      @io = io
+      @line = ""
+    end
+  
+    def gets(*args)
+      if args.count == 1 && args[0].is_a?(String)
+        delim = args[0]
+        @line = "" if @new_line
+        s = @io.gets(delim)
+        if s != nil
+          @line << s 
+        end
+        return s
+      else
+        @io.gets(*args)
+      end
+    end
+  
+    def eof?
+      @io.eof?
+    end
+  
+    def finished
+      @new_line = true
+    end
+  
+    def line
+      @line
+    end
+    
+    def method_missing(method, *args)
+      @io.send(method, *args)
+    end
+    
+  end
+end