wjordan213-csvlint 0.2.8

data/features/validation_warnings.feature

@@ -0,0 +1,86 @@
+Feature: Validation warnings
+
+  Scenario: UTF-8 Encoding
+    Given I have a CSV with the following content:
+    """
+"col1","col2","col3"
+"abc","2","3"
+    """
+    And it is encoded as "utf-8"
+    And it is stored at the url "http://example.com/example1.csv"
+    When I ask if there are warnings
+    Then there should be 0 warnings
+
+   Scenario: ISO-8859-1 Encoding
+    Given I have a CSV with the following content:
+    """
+"col1","col2","col3"
+"1","2","3"
+    """
+     And it is encoded as "iso-8859-1"
+    And it is stored at the url "http://example.com/example1.csv"
+    When I ask if there are warnings
+    Then there should be 1 warnings
+
+  Scenario: Correct content type
+    Given I have a CSV with the following content:
+    """
+"col1","col2","col3"
+"abc","2","3"
+    """
+    And the content type is set to "text/csv"
+    And it is stored at the url "http://example.com/example1.csv"
+    And I ask if there are warnings
+    Then there should be 0 warnings
+
+  Scenario: No extension
+    Given I have a CSV with the following content:
+    """
+"col1","col2","col3"
+"abc","2","3"
+    """
+    And the content type is set to "text/csv"
+    And it is stored at the url "http://example.com/example1"
+    And I ask if there are warnings
+    Then there should be 0 warnings
+
+  Scenario: Allow query params after extension
+    Given I have a CSV with the following content:
+    """
+"col1","col2","col3"
+"abc","2","3"
+    """
+    And the content type is set to "text/csv"
+    And it is stored at the url "http://example.com/example1.csv?query=param"
+    And I ask if there are warnings
+    Then there should be 0 warnings
+
+  Scenario: User doesn't supply encoding
+    Given I have a CSV with the following content:
+    """
+"col1","col2","col3"
+"abc","2","3"
+    """
+    And it is stored at the url "http://example.com/example1.csv" with no character set
+    When I ask if there are warnings
+    Then there should be 1 warnings
+    And that warning should have the type "no_encoding"
+
+  Scenario: Title rows
+    Given I have a CSV file called "title-row.csv"
+    And it is stored at the url "http://example.com/example1.csv"
+    And I ask if there are warnings
+    Then there should be 1 warnings
+    And that warning should have the type "title_row"
+
+  Scenario: catch excel warnings
+    Given I parse a file called "spreadsheet.xls"
+    And I ask if there are warnings
+    Then there should be 1 warnings
+    And that warning should have the type "excel"
+
+  Scenario: catch excel warnings
+    Given I parse a file called "spreadsheet.xlsx"
+    And I ask if there are warnings
+    Then there should be 1 warnings
+    And that warning should have the type "excel"

data/lib/csvlint.rb

@@ -0,0 +1,27 @@
+require 'csv'
+require 'date'
+require 'open-uri'
+require 'set'
+require 'tempfile'
+require 'typhoeus'
+
+require 'active_support/core_ext/date/conversions'
+require 'active_support/core_ext/time/conversions'
+require 'mime/types'
+require 'open_uri_redirections'
+require 'uri_template'
+
+require 'csvlint/error_message'
+require 'csvlint/error_collector'
+require 'csvlint/validate'
+require 'csvlint/field'
+
+require 'csvlint/csvw/metadata_error'
+require 'csvlint/csvw/number_format'
+require 'csvlint/csvw/date_format'
+require 'csvlint/csvw/property_checker'
+require 'csvlint/csvw/column'
+require 'csvlint/csvw/table'
+require 'csvlint/csvw/table_group'
+
+require 'csvlint/schema'

data/lib/csvlint/cli.rb

@@ -0,0 +1,165 @@
+require 'csvlint'
+require 'colorize'
+require 'json'
+require 'pp'
+require 'thor'
+
+module Csvlint
+  class Cli < Thor
+
+    desc "myfile.csv OR csvlint http://example.com/myfile.csv", "Supports validating CSV files to check their syntax and contents"
+    option :dump_errors, desc: "Pretty print error and warning objects.", type: :boolean, aliases: :d
+    option :schema, banner: "FILENAME OR URL", desc: "Schema file", aliases: :s
+    def validate(source = nil)
+      source = read_source(source)
+      @schema = get_schema(options[:schema]) if options[:schema]
+      fetch_schema_tables(@schema, options) if source.nil?
+
+      valid = validate_csv(source, @schema, options[:dump])
+      exit 1 unless valid
+    end
+
+    def help
+      self.class.command_help(shell, :validate)
+    end
+
+    default_task :validate
+
+    private
+
+      def read_source(source)
+        if source.nil?
+          # If no source is present, try reading from stdin
+          if !$stdin.tty?
+            source = StringIO.new(STDIN.read) rescue nil
+            return_error "No CSV data to validate" if !options[:schema] && source.nil?
+          end
+        else
+          # If the source isn't a URL, it's a file
+          unless source =~ /^http(s)?/
+            begin
+              source = File.new( source )
+            rescue Errno::ENOENT
+              return_error "#{source} not found"
+            end
+          end
+        end
+
+        source
+      end
+
+      def get_schema(schema)
+        begin
+          schema = Csvlint::Schema.load_from_json(schema, false)
+        rescue Csvlint::Csvw::MetadataError => e
+          return_error "invalid metadata: #{e.message}#{" at " + e.path if e.path}"
+        rescue OpenURI::HTTPError, Errno::ENOENT
+          return_error "#{options[:schema]} not found"
+        end
+
+        if schema.class == Csvlint::Schema && schema.description == "malformed"
+          return_error "invalid metadata: malformed JSON"
+        end
+
+        schema
+      end
+
+      def fetch_schema_tables(schema, options)
+        valid = true
+
+        unless schema.instance_of? Csvlint::Csvw::TableGroup
+          return_error "No CSV data to validate."
+        end
+        schema.tables.keys.each do |source|
+          begin
+            source = source.sub("file:","")
+            source = File.new( source )
+          rescue Errno::ENOENT
+            return_error "#{source} not found"
+          end unless source =~ /^http(s)?/
+          valid &= validate_csv(source, schema, options[:dump])
+        end
+
+        exit 1 unless valid
+      end
+
+      def print_error(index, error, dump, color)
+        location = ""
+        location += error.row.to_s if error.row
+        location += "#{error.row ? "," : ""}#{error.column.to_s}" if error.column
+        if error.row || error.column
+          location = "#{error.row ? "Row" : "Column"}: #{location}"
+        end
+        output_string = "#{index+1}. "
+        if error.column && @schema && @schema.class == Csvlint::Schema
+          output_string += "#{@schema.fields[error.column - 1].name}: "
+        end
+        output_string += "#{error.type}"
+        output_string += ". #{location}" unless location.empty?
+        output_string += ". #{error.content}" if error.content
+
+        if $stdout.tty?
+          puts output_string.colorize(color)
+        else
+          puts output_string
+        end
+
+        if dump
+          pp error
+        end
+      end
+
+      def print_errors(errors, dump)
+        if errors.size > 0
+          errors.each_with_index { |error, i| print_error(i, error, dump, :red)  }
+        end
+      end
+
+      def return_error(message)
+        if $stdout.tty?
+          puts message.colorize(:red)
+        else
+          puts message
+        end
+        exit 1
+      end
+
+      def validate_csv(source, schema, dump)
+        @error_count = 0
+
+        validator = Csvlint::Validator.new( source, {}, schema, { lambda: report_lines } )
+
+        if source.class == String
+          csv = source
+        elsif source.class == File
+          csv = source.path
+        else
+          csv = "CSV"
+        end
+
+        if $stdout.tty?
+          puts "\r\n#{csv} is #{validator.valid? ? "VALID".green : "INVALID".red}"
+        else
+          puts "\r\n#{csv} is #{validator.valid? ? "VALID" : "INVALID"}"
+        end
+
+        print_errors(validator.errors, dump)
+        print_errors(validator.warnings, dump)
+
+        return validator.valid?
+      end
+
+      def report_lines
+        lambda do |row|
+          new_errors = row.errors.count
+          if new_errors > @error_count
+            print "!".red
+          else
+            print ".".green
+          end
+          @error_count = new_errors
+        end
+      end
+
+  end
+end

data/lib/csvlint/csvw/column.rb

@@ -0,0 +1,359 @@
+module Csvlint
+  module Csvw
+    class Column
+      include Csvlint::ErrorCollector
+
+      attr_reader :id, :about_url, :datatype, :default, :lang, :name, :null, :number, :ordered, :property_url, :required, :separator, :source_number, :suppress_output, :text_direction, :titles, :value_url, :virtual, :annotations
+
+      def initialize(number, name, id: nil, about_url: nil, datatype: { "@id" => "http://www.w3.org/2001/XMLSchema#string" }, default: "", lang: "und", null: [""], ordered: false, property_url: nil, required: false, separator: nil, source_number: nil, suppress_output: false, text_direction: :inherit, titles: {}, value_url: nil, virtual: false, annotations: [], warnings: [])
+        @number = number
+        @name = name
+        @id = id
+        @about_url = about_url
+        @datatype = datatype
+        @default = default
+        @lang = lang
+        @null = null
+        @ordered = ordered
+        @property_url = property_url
+        @required = required
+        @separator = separator
+        @source_number = source_number || number
+        @suppress_output = suppress_output
+        @text_direction = text_direction
+        @titles = titles
+        @value_url = value_url
+        @virtual = virtual
+        @annotations = annotations
+        reset
+        @warnings += warnings
+      end
+
+      def self.from_json(number, column_desc, base_url=nil, lang="und", inherited_properties={})
+        annotations = {}
+        warnings = []
+        column_properties = {}
+        inherited_properties = inherited_properties.clone
+
+        column_desc.each do |property,value|
+          if property == "@type"
+            raise Csvlint::Csvw::MetadataError.new("columns[#{number}].@type"), "@type of column is not 'Column'" if value != 'Column'
+          else
+            v, warning, type = Csvw::PropertyChecker.check_property(property, value, base_url, lang)
+            warnings += Array(warning).map{ |w| Csvlint::ErrorMessage.new(w, :metadata, nil, nil, "#{property}: #{value}", nil) } unless warning.nil? || warning.empty?
+            if type == :annotation
+              annotations[property] = v
+            elsif type == :common || type == :column
+              column_properties[property] = v
+            elsif type == :inherited
+              inherited_properties[property] = v
+            else
+              warnings << Csvlint::ErrorMessage.new(:invalid_property, :metadata, nil, nil, "column: #{property}", nil)
+            end
+          end
+        end
+
+        return self.new(number, column_properties["name"],
+          id: column_properties["@id"],
+          datatype: inherited_properties["datatype"] || { "@id" => "http://www.w3.org/2001/XMLSchema#string" },
+          lang: inherited_properties["lang"] || "und",
+          null: inherited_properties["null"] || [""],
+          property_url: column_desc["propertyUrl"],
+          required: inherited_properties["required"] || false,
+          separator: inherited_properties["separator"],
+          titles: column_properties["titles"],
+          virtual: column_properties["virtual"] || false,
+          annotations: annotations,
+          warnings: warnings
+        )
+      end
+
+      def validate_header(header)
+        reset
+        valid_headers = @titles ? @titles.map{ |l,v| v if Column.languages_match(l, lang) }.flatten : []
+        build_errors(:invalid_header, :schema, 1, @number, header, @titles) unless valid_headers.include? header
+        return valid?
+      end
+
+      def validate(string_value, row=nil)
+        reset
+        values = parse(string_value || "", row)
+        # STDERR.puts "#{name} - #{string_value.inspect} - #{values.inspect}"
+        values.each do |value|
+          validate_required(value, row)
+          validate_format(value, row)
+          validate_length(value, row)
+          validate_value(value, row)
+        end unless values.nil?
+        validate_required(values, row) if values.nil?
+        return valid?
+      end
+
+      def parse(string_value, row=nil)
+        return nil if null.include? string_value
+        string_values = @separator.nil? ? [string_value] : string_value.split(@separator)
+        values = []
+        string_values.each do |s|
+          value, warning = DATATYPE_PARSER[@datatype["base"] || @datatype["@id"]].call(s, @datatype["format"])
+          if warning.nil?
+            values << value
+          else
+            build_errors(warning, :schema, row, @number, s, @datatype)
+            values << s
+          end
+        end
+        return values
+      end
+
+      private
+        class << self
+
+          def create_date_parser(type, warning)
+            return lambda { |value, format|
+              format = Csvlint::Csvw::DateFormat.new(nil, type) if format.nil?
+              v = format.parse(value)
+              return nil, warning if v.nil?
+              return v, nil
+            }
+          end
+
+          def create_regexp_based_parser(regexp, warning)
+            return lambda { |value, format|
+              return nil, warning unless value =~ regexp
+              return value, nil
+            }
+          end
+
+          def languages_match(l1, l2)
+            return true if l1 == l2 || l1 == "und" || l2 == "und"
+            return true if l1 =~ Regexp.new("^#{l2}-") || l2 =~ Regexp.new("^#{l1}-")
+            return false
+          end
+        end
+
+        def validate_required(value, row)
+          build_errors(:required, :schema, row, number, value, { "required" => @required }) if @required && value.nil?
+        end
+
+        def validate_length(value, row)
+          if datatype["length"] || datatype["minLength"] || datatype["maxLength"]
+            length = value.length
+            length = value.gsub(/==?$/,"").length * 3 / 4 if datatype["@id"] == "http://www.w3.org/2001/XMLSchema#base64Binary" || datatype["base"] == "http://www.w3.org/2001/XMLSchema#base64Binary"
+            length = value.length / 2 if datatype["@id"] == "http://www.w3.org/2001/XMLSchema#hexBinary" || datatype["base"] == "http://www.w3.org/2001/XMLSchema#hexBinary"
+
+            build_errors(:min_length, :schema, row, number, value, { "minLength" => datatype["minLength"] }) if datatype["minLength"] && length < datatype["minLength"]
+            build_errors(:max_length, :schema, row, number, value, { "maxLength" => datatype["maxLength"] }) if datatype["maxLength"] && length > datatype["maxLength"]
+            build_errors(:length, :schema, row, number, value, { "length" => datatype["length"] }) if datatype["length"] && length != datatype["length"]
+          end
+        end
+
+        def validate_format(value, row)
+          if datatype["format"]
+            build_errors(:format, :schema, row, number, value, { "format" => datatype["format"] }) unless DATATYPE_FORMAT_VALIDATION[datatype["base"]].call(value, datatype["format"])
+          end
+        end
+
+        def validate_value(value, row)
+          build_errors(:min_inclusive, :schema, row, number, value, { "minInclusive" => datatype["minInclusive"] }) if datatype["minInclusive"] && value < datatype["minInclusive"]
+          build_errors(:max_inclusive, :schema, row, number, value, { "maxInclusive" => datatype["maxInclusive"] }) if datatype["maxInclusive"] && value > datatype["maxInclusive"]
+          build_errors(:min_exclusive, :schema, row, number, value, { "minExclusive" => datatype["minExclusive"] }) if datatype["minExclusive"] && value <= datatype["minExclusive"]
+          build_errors(:max_exclusive, :schema, row, number, value, { "maxExclusive" => datatype["maxExclusive"] }) if datatype["maxExclusive"] && value >= datatype["maxExclusive"]
+        end
+
+        REGEXP_VALIDATION = lambda { |value, format| value =~ format }
+
+        NO_ADDITIONAL_VALIDATION = lambda { |value, format| true }
+
+        DATATYPE_FORMAT_VALIDATION = {
+          "http://www.w3.org/1999/02/22-rdf-syntax-ns#XMLLiteral" => REGEXP_VALIDATION,
+          "http://www.w3.org/1999/02/22-rdf-syntax-ns#HTML" => REGEXP_VALIDATION,
+          "http://www.w3.org/ns/csvw#JSON" => REGEXP_VALIDATION,
+          "http://www.w3.org/2001/XMLSchema#anyAtomicType" => REGEXP_VALIDATION,
+          "http://www.w3.org/2001/XMLSchema#anyURI" => REGEXP_VALIDATION,
+          "http://www.w3.org/2001/XMLSchema#base64Binary" => REGEXP_VALIDATION,
+          "http://www.w3.org/2001/XMLSchema#boolean" => NO_ADDITIONAL_VALIDATION,
+          "http://www.w3.org/2001/XMLSchema#date" => NO_ADDITIONAL_VALIDATION,
+          "http://www.w3.org/2001/XMLSchema#dateTime" => NO_ADDITIONAL_VALIDATION,
+          "http://www.w3.org/2001/XMLSchema#dateTimeStamp" => NO_ADDITIONAL_VALIDATION,
+          "http://www.w3.org/2001/XMLSchema#decimal" => NO_ADDITIONAL_VALIDATION,
+          "http://www.w3.org/2001/XMLSchema#integer" => NO_ADDITIONAL_VALIDATION,
+          "http://www.w3.org/2001/XMLSchema#long" => NO_ADDITIONAL_VALIDATION,
+          "http://www.w3.org/2001/XMLSchema#int" => NO_ADDITIONAL_VALIDATION,
+          "http://www.w3.org/2001/XMLSchema#short" => NO_ADDITIONAL_VALIDATION,
+          "http://www.w3.org/2001/XMLSchema#byte" => NO_ADDITIONAL_VALIDATION,
+          "http://www.w3.org/2001/XMLSchema#nonNegativeInteger" => NO_ADDITIONAL_VALIDATION,
+          "http://www.w3.org/2001/XMLSchema#positiveInteger" => NO_ADDITIONAL_VALIDATION,
+          "http://www.w3.org/2001/XMLSchema#unsignedLong" => NO_ADDITIONAL_VALIDATION,
+          "http://www.w3.org/2001/XMLSchema#unsignedInt" => NO_ADDITIONAL_VALIDATION,
+          "http://www.w3.org/2001/XMLSchema#unsignedShort" => NO_ADDITIONAL_VALIDATION,
+          "http://www.w3.org/2001/XMLSchema#unsignedByte" => NO_ADDITIONAL_VALIDATION,
+          "http://www.w3.org/2001/XMLSchema#nonPositiveInteger" => NO_ADDITIONAL_VALIDATION,
+          "http://www.w3.org/2001/XMLSchema#negativeInteger" => NO_ADDITIONAL_VALIDATION,
+          "http://www.w3.org/2001/XMLSchema#double" => NO_ADDITIONAL_VALIDATION,
+          "http://www.w3.org/2001/XMLSchema#duration" => REGEXP_VALIDATION,
+          "http://www.w3.org/2001/XMLSchema#dayTimeDuration" => REGEXP_VALIDATION,
+          "http://www.w3.org/2001/XMLSchema#yearMonthDuration" => REGEXP_VALIDATION,
+          "http://www.w3.org/2001/XMLSchema#float" => NO_ADDITIONAL_VALIDATION,
+          "http://www.w3.org/2001/XMLSchema#gDay" => NO_ADDITIONAL_VALIDATION,
+          "http://www.w3.org/2001/XMLSchema#gMonth" => NO_ADDITIONAL_VALIDATION,
+          "http://www.w3.org/2001/XMLSchema#gMonthDay" => NO_ADDITIONAL_VALIDATION,
+          "http://www.w3.org/2001/XMLSchema#gYear" => NO_ADDITIONAL_VALIDATION,
+          "http://www.w3.org/2001/XMLSchema#gYearMonth" => NO_ADDITIONAL_VALIDATION,
+          "http://www.w3.org/2001/XMLSchema#hexBinary" => REGEXP_VALIDATION,
+          "http://www.w3.org/2001/XMLSchema#QName" => REGEXP_VALIDATION,
+          "http://www.w3.org/2001/XMLSchema#string" => REGEXP_VALIDATION,
+          "http://www.w3.org/2001/XMLSchema#normalizedString" => REGEXP_VALIDATION,
+          "http://www.w3.org/2001/XMLSchema#token" => REGEXP_VALIDATION,
+          "http://www.w3.org/2001/XMLSchema#language" => REGEXP_VALIDATION,
+          "http://www.w3.org/2001/XMLSchema#Name" => REGEXP_VALIDATION,
+          "http://www.w3.org/2001/XMLSchema#NMTOKEN" => REGEXP_VALIDATION,
+          "http://www.w3.org/2001/XMLSchema#time" => NO_ADDITIONAL_VALIDATION
+        }
+
+        ALL_VALUES_VALID = lambda { |value, format| return value, nil }
+
+        NUMERIC_PARSER = lambda { |value, format|
+          format = Csvlint::Csvw::NumberFormat.new() if format.nil?
+          v = format.parse(value)
+          return nil, :invalid_number if v.nil?
+          return v, nil
+        }
+
+        DATATYPE_PARSER = {
+          "http://www.w3.org/1999/02/22-rdf-syntax-ns#XMLLiteral" => ALL_VALUES_VALID,
+          "http://www.w3.org/1999/02/22-rdf-syntax-ns#HTML" => ALL_VALUES_VALID,
+          "http://www.w3.org/ns/csvw#JSON" => ALL_VALUES_VALID,
+          "http://www.w3.org/2001/XMLSchema#anyAtomicType" => ALL_VALUES_VALID,
+          "http://www.w3.org/2001/XMLSchema#anyURI" => ALL_VALUES_VALID,
+          "http://www.w3.org/2001/XMLSchema#base64Binary" => ALL_VALUES_VALID,
+          "http://www.w3.org/2001/XMLSchema#boolean" => lambda { |value, format|
+            if format.nil?
+              return true, nil if ["true", "1"].include? value
+              return false, nil if ["false", "0"].include? value
+            else
+              return true, nil if value == format[0]
+              return false, nil if value == format[1]
+            end
+            return value, :invalid_boolean
+          },
+          "http://www.w3.org/2001/XMLSchema#date" =>
+            create_date_parser("http://www.w3.org/2001/XMLSchema#date", :invalid_date),
+          "http://www.w3.org/2001/XMLSchema#dateTime" =>
+            create_date_parser("http://www.w3.org/2001/XMLSchema#dateTime", :invalid_date_time),
+          "http://www.w3.org/2001/XMLSchema#dateTimeStamp" =>
+            create_date_parser("http://www.w3.org/2001/XMLSchema#dateTimeStamp", :invalid_date_time_stamp),
+          "http://www.w3.org/2001/XMLSchema#decimal" => lambda { |value, format|
+            return nil, :invalid_decimal if value =~ /(E|^(NaN|INF|-INF)$)/
+            return NUMERIC_PARSER.call(value, format)
+          },
+          "http://www.w3.org/2001/XMLSchema#integer" => lambda { |value, format|
+            v, w = NUMERIC_PARSER.call(value, format)
+            return v, :invalid_integer unless w.nil?
+            return nil, :invalid_integer unless v.kind_of? Integer
+            return v, w
+          },
+          "http://www.w3.org/2001/XMLSchema#long" => lambda { |value, format|
+            v, w = DATATYPE_PARSER["http://www.w3.org/2001/XMLSchema#integer"].call(value, format)
+            return v, :invalid_long unless w.nil?
+            return nil, :invalid_long unless v <= 9223372036854775807 && v >= -9223372036854775808
+            return v, w
+          },
+          "http://www.w3.org/2001/XMLSchema#int" => lambda { |value, format|
+            v, w = DATATYPE_PARSER["http://www.w3.org/2001/XMLSchema#integer"].call(value, format)
+            return v, :invalid_int unless w.nil?
+            return nil, :invalid_int unless v <= 2147483647 && v >= -2147483648
+            return v, w
+          },
+          "http://www.w3.org/2001/XMLSchema#short" => lambda { |value, format|
+            v, w = DATATYPE_PARSER["http://www.w3.org/2001/XMLSchema#integer"].call(value, format)
+            return v, :invalid_short unless w.nil?
+            return nil, :invalid_short unless v <= 32767 && v >= -32768
+            return v, w
+          },
+          "http://www.w3.org/2001/XMLSchema#byte" => lambda { |value, format|
+            v, w = DATATYPE_PARSER["http://www.w3.org/2001/XMLSchema#integer"].call(value, format)
+            return v, :invalid_byte unless w.nil?
+            return nil, :invalid_byte unless v <= 127 && v >= -128
+            return v, w
+          },
+          "http://www.w3.org/2001/XMLSchema#nonNegativeInteger" => lambda { |value, format|
+            v, w = DATATYPE_PARSER["http://www.w3.org/2001/XMLSchema#integer"].call(value, format)
+            return v, :invalid_nonNegativeInteger unless w.nil?
+            return nil, :invalid_nonNegativeInteger unless v >= 0
+            return v, w
+          },
+          "http://www.w3.org/2001/XMLSchema#positiveInteger" => lambda { |value, format|
+            v, w = DATATYPE_PARSER["http://www.w3.org/2001/XMLSchema#integer"].call(value, format)
+            return v, :invalid_positiveInteger unless w.nil?
+            return nil, :invalid_positiveInteger unless v > 0
+            return v, w
+          },
+          "http://www.w3.org/2001/XMLSchema#unsignedLong" => lambda { |value, format|
+            v, w = DATATYPE_PARSER["http://www.w3.org/2001/XMLSchema#nonNegativeInteger"].call(value, format)
+            return v, :invalid_unsignedLong unless w.nil?
+            return nil, :invalid_unsignedLong unless v <= 18446744073709551615
+            return v, w
+          },
+          "http://www.w3.org/2001/XMLSchema#unsignedInt" => lambda { |value, format|
+            v, w = DATATYPE_PARSER["http://www.w3.org/2001/XMLSchema#nonNegativeInteger"].call(value, format)
+            return v, :invalid_unsignedInt unless w.nil?
+            return nil, :invalid_unsignedInt unless v <= 4294967295
+            return v, w
+          },
+          "http://www.w3.org/2001/XMLSchema#unsignedShort" => lambda { |value, format|
+            v, w = DATATYPE_PARSER["http://www.w3.org/2001/XMLSchema#nonNegativeInteger"].call(value, format)
+            return v, :invalid_unsignedShort unless w.nil?
+            return nil, :invalid_unsignedShort unless v <= 65535
+            return v, w
+          },
+          "http://www.w3.org/2001/XMLSchema#unsignedByte" => lambda { |value, format|
+            v, w = DATATYPE_PARSER["http://www.w3.org/2001/XMLSchema#nonNegativeInteger"].call(value, format)
+            return v, :invalid_unsignedByte unless w.nil?
+            return nil, :invalid_unsignedByte unless v <= 255
+            return v, w
+          },
+          "http://www.w3.org/2001/XMLSchema#nonPositiveInteger" => lambda { |value, format|
+            v, w = DATATYPE_PARSER["http://www.w3.org/2001/XMLSchema#integer"].call(value, format)
+            return v, :invalid_nonPositiveInteger unless w.nil?
+            return nil, :invalid_nonPositiveInteger unless v <= 0
+            return v, w
+          },
+          "http://www.w3.org/2001/XMLSchema#negativeInteger" => lambda { |value, format|
+            v, w = DATATYPE_PARSER["http://www.w3.org/2001/XMLSchema#integer"].call(value, format)
+            return v, :invalid_negativeInteger unless w.nil?
+            return nil, :invalid_negativeInteger unless v < 0
+            return v, w
+          },
+          "http://www.w3.org/2001/XMLSchema#double" => NUMERIC_PARSER,
+          # regular expressions here taken from XML Schema datatypes spec
+          "http://www.w3.org/2001/XMLSchema#duration" =>
+            create_regexp_based_parser(/-?P((([0-9]+Y([0-9]+M)?([0-9]+D)?|([0-9]+M)([0-9]+D)?|([0-9]+D))(T(([0-9]+H)([0-9]+M)?([0-9]+(\.[0-9]+)?S)?|([0-9]+M)([0-9]+(\.[0-9]+)?S)?|([0-9]+(\.[0-9]+)?S)))?)|(T(([0-9]+H)([0-9]+M)?([0-9]+(\.[0-9]+)?S)?|([0-9]+M)([0-9]+(\.[0-9]+)?S)?|([0-9]+(\.[0-9]+)?S))))/, :invalid_duration),
+          "http://www.w3.org/2001/XMLSchema#dayTimeDuration" =>
+            create_regexp_based_parser(/-?P(([0-9]+D(T(([0-9]+H)([0-9]+M)?([0-9]+(\.[0-9]+)?S)?|([0-9]+M)([0-9]+(\.[0-9]+)?S)?|([0-9]+(\.[0-9]+)?S)))?)|(T(([0-9]+H)([0-9]+M)?([0-9]+(\.[0-9]+)?S)?|([0-9]+M)([0-9]+(\.[0-9]+)?S)?|([0-9]+(\.[0-9]+)?S))))/, :invalid_dayTimeDuration),
+          "http://www.w3.org/2001/XMLSchema#yearMonthDuration" =>
+            create_regexp_based_parser(/-?P([0-9]+Y([0-9]+M)?|([0-9]+M))/, :invalid_duration),
+          "http://www.w3.org/2001/XMLSchema#float" => NUMERIC_PARSER,
+          "http://www.w3.org/2001/XMLSchema#gDay" =>
+            create_date_parser("http://www.w3.org/2001/XMLSchema#gDay", :invalid_gDay),
+          "http://www.w3.org/2001/XMLSchema#gMonth" =>
+            create_date_parser("http://www.w3.org/2001/XMLSchema#gMonth", :invalid_gMonth),
+          "http://www.w3.org/2001/XMLSchema#gMonthDay" =>
+            create_date_parser("http://www.w3.org/2001/XMLSchema#gMonthDay", :invalid_gMonthDay),
+          "http://www.w3.org/2001/XMLSchema#gYear" =>
+            create_date_parser("http://www.w3.org/2001/XMLSchema#gYear", :invalid_gYear),
+          "http://www.w3.org/2001/XMLSchema#gYearMonth" =>
+            create_date_parser("http://www.w3.org/2001/XMLSchema#gYearMonth", :invalid_gYearMonth),
+          "http://www.w3.org/2001/XMLSchema#hexBinary" => ALL_VALUES_VALID,
+          "http://www.w3.org/2001/XMLSchema#QName" => ALL_VALUES_VALID,
+          "http://www.w3.org/2001/XMLSchema#string" => ALL_VALUES_VALID,
+          "http://www.w3.org/2001/XMLSchema#normalizedString" => ALL_VALUES_VALID,
+          "http://www.w3.org/2001/XMLSchema#token" => ALL_VALUES_VALID,
+          "http://www.w3.org/2001/XMLSchema#language" => ALL_VALUES_VALID,
+          "http://www.w3.org/2001/XMLSchema#Name" => ALL_VALUES_VALID,
+          "http://www.w3.org/2001/XMLSchema#NMTOKEN" => ALL_VALUES_VALID,
+          "http://www.w3.org/2001/XMLSchema#time" =>
+            create_date_parser("http://www.w3.org/2001/XMLSchema#time", :invalid_time)
+        }
+    end
+  end
+end