csvlint 0.2.6 → 0.3.0

data/lib/csvlint/csvw/date_format.rb

@@ -70,39 +70,72 @@ module Csvlint
         # STDERR.puts(@regexp)
         # STDERR.puts(value)
         # STDERR.puts(match.inspect)
+        value = {}
+        match.names.each do |field|
+          unless match[field].nil?
+            case field
+            when "timezone"
+              tz = match["timezone"]
+              tz = "+00:00" if tz == 'Z'
+              tz += ':00' if tz.length == 3
+              tz = "#{tz[0..2]}:#{tz[3..4]}" unless tz =~ /:/
+              value[:timezone] = tz
+            when "second"
+              value[:second] = match["second"].to_f
+            else
+              value[field.to_sym] = match[field].to_i
+            end
+          end
+        end
         case @type
         when "http://www.w3.org/2001/XMLSchema#date"
           begin
-            return Date.new(match["year"].to_i, match["month"].to_i, match["day"].to_i)
+            value[:dateTime] = Date.new(match["year"].to_i, match["month"].to_i, match["day"].to_i)
           rescue ArgumentError
             return nil
           end
         when "http://www.w3.org/2001/XMLSchema#dateTime"
           begin
-            return DateTime.new(match["year"].to_i, match["month"].to_i, match["day"].to_i, match["hour"].to_i, match["minute"].to_i, (match.names.include?("second") ? match["second"].to_f : 0), match.names.include?("timezone") && match["timezone"] ? match["timezone"] : '')
+            value[:dateTime] = DateTime.new(match["year"].to_i, match["month"].to_i, match["day"].to_i, match["hour"].to_i, match["minute"].to_i, (match.names.include?("second") ? match["second"].to_f : 0), match.names.include?("timezone") && match["timezone"] ? match["timezone"] : '')
           rescue ArgumentError
             return nil
           end
         else
-          value = {}
-          match.names.each do |field|
-            unless match[field].nil?
-              case field
-              when "timezone"
-                tz = match["timezone"]
-                tz = "+00:00" if tz == 'Z'
-                tz += ':00' if tz.length == 3
-                tz = "#{tz[0..2]}:#{tz[3..4]}" unless tz =~ /:/
-                value["timezone"] = tz
-              when "second"
-                value["second"] = match["second"].to_f
+          value[:dateTime] = DateTime.new(value[:year] || 0, value[:month] || 1, value[:day] || 1, value[:hour] || 0, value[:minute] || 0, value[:second] || 0, value[:timezone] || "+00:00")
+        end
+        if value[:year]
+          if value[:month]
+            if value[:day]
+              if value[:hour]
+                # dateTime
+                value[:string] = "#{format('%04d', value[:year])}-#{format('%02d', value[:month])}-#{format('%02d', value[:day])}T#{format('%02d', value[:hour])}:#{format('%02d', value[:minute] || 0)}:#{format('%02g', value[:second] || 0)}#{value[:timezone] ? value[:timezone].sub("+00:00", "Z") : ''}"
               else
-                value[field] = match[field].to_i
+                # date
+                value[:string] = "#{format('%04d', value[:year])}-#{format('%02d', value[:month])}-#{format('%02d', value[:day])}#{value[:timezone] ? value[:timezone].sub("+00:00", "Z") : ''}"
               end
+            else
+              # gYearMonth
+              value[:string] = "#{format('%04d', value[:year])}-#{format('%02d', value[:month])}#{value[:timezone] ? value[:timezone].sub("+00:00", "Z") : ''}"
             end
+          else
+            # gYear
+            value[:string] = "#{format('%04d', value[:year])}#{value[:timezone] ? value[:timezone].sub("+00:00", "Z") : ''}"
           end
-          return value
+        elsif value[:month]
+          if value[:day]
+            # gMonthDay
+            value[:string] = "--#{format('%02d', value[:month])}-#{format('%02d', value[:day])}#{value[:timezone] ? value[:timezone].sub("+00:00", "Z") : ''}"
+          else
+            # gMonth
+            value[:string] = "--#{format('%02d', value[:month])}#{value[:timezone] ? value[:timezone].sub("+00:00", "Z") : ''}"
+          end
+        elsif value[:day]
+          # gDay
+          value[:string] = "---#{format('%02d', value[:day])}#{value[:timezone] ? value[:timezone].sub("+00:00", "Z") : ''}"
+        else
+          value[:string] = "#{format('%02d', value[:hour])}:#{format('%02d', value[:minute])}:#{format('%02g', value[:second] || 0)}#{value[:timezone] ? value[:timezone].sub("+00:00", "Z") : ''}"
         end
+        return value
       end
 
       private

data/lib/csvlint/csvw/number_format.rb

@@ -2,16 +2,28 @@ module Csvlint
   module Csvw
     class NumberFormat
 
-      attr_reader :pattern, :prefix, :numeric_part, :suffix, :grouping_separator, :decimal_separator, :primary_grouping_size, :secondary_grouping_size, :fractional_grouping_size
+      attr_reader :integer, :pattern, :prefix, :numeric_part, :suffix, :grouping_separator, :decimal_separator, :primary_grouping_size, :secondary_grouping_size, :fractional_grouping_size
 
-      def initialize(pattern=nil, grouping_separator=nil, decimal_separator=".")
+      def initialize(pattern=nil, grouping_separator=nil, decimal_separator=".", integer=nil)
         @pattern = pattern
+        @integer = integer
+        if @integer.nil?
+          if @pattern.nil?
+            @integer = nil
+          else
+            @integer = !@pattern.include?(decimal_separator)
+          end
+        end
         @grouping_separator = grouping_separator || (@pattern.nil? ? nil : ",")
         @decimal_separator = decimal_separator || "."
         if pattern.nil?
-          @regexp = Regexp.new("^(([-+]?[0-9]+(#{Regexp.escape(@decimal_separator)}[0-9]+)?([Ee][-+]?[0-9]+)?[%‰]?)|NaN|INF|-INF)$")
+          if integer
+            @regexp = INTEGER_REGEXP
+          else
+            @regexp = Regexp.new("^(([-+]?[0-9]+(\\.[0-9]+)?([Ee][-+]?[0-9]+)?[%‰]?)|NaN|INF|-INF)$")
+          end
         else
-          numeric_part_regexp = Regexp.new("(?<numeric_part>([0#Ee]|#{Regexp.escape(@grouping_separator)}|#{Regexp.escape(@decimal_separator)})+)")
+          numeric_part_regexp = Regexp.new("(?<numeric_part>[-+]?([0#Ee]|#{Regexp.escape(@grouping_separator)}|#{Regexp.escape(@decimal_separator)})+)")
           number_format_regexp = Regexp.new("^(?<prefix>.*?)#{numeric_part_regexp}(?<suffix>.*?)$")
           match = number_format_regexp.match(pattern)
           raise Csvw::NumberFormatError, "invalid number format" if match.nil?
@@ -28,7 +40,12 @@ module Csvlint
           integer_part = mantissa_parts[0]
           fractional_part = mantissa_parts[1] || ""
 
-          @integer_pattern = exponent_part == "" && fractional_part == ""
+          if ["+", "-"].include?(integer_part[0])
+            numeric_part_regexp = "\\#{integer_part[0]}"
+            integer_part = integer_part[1..-1]
+          else
+            numeric_part_regexp = "[-+]?"
+          end
 
           min_integer_digits = integer_part.gsub(@grouping_separator, "").gsub("#", "").length
           min_fraction_digits = fractional_part.gsub(@grouping_separator, "").gsub("#", "").length
@@ -43,8 +60,6 @@ module Csvlint
           fractional_parts = fractional_part.split(@grouping_separator)[0..-2]
           @fractional_grouping_size = fractional_parts[0].length rescue 0
 
-          numeric_part_regexp = "[-+]?"
-
           if @primary_grouping_size == 0
             integer_regexp = "[0-9]*[0-9]{#{min_integer_digits}}"
           else
@@ -160,16 +175,8 @@ module Csvlint
         if @pattern.nil?
           return nil if !@grouping_separator.nil? && value =~ Regexp.new("((^#{Regexp.escape(@grouping_separator)})|#{Regexp.escape(@grouping_separator)}{2})")
           value.gsub!(@grouping_separator, "") unless @grouping_separator.nil?
-          if value =~ INTEGER_REGEXP
-            case value[-1]
-            when "%"
-              return value.to_f / 100
-            when "‰"
-              return value.to_f / 1000
-            else
-              return value.to_i
-            end
-          elsif value =~ @regexp
+          value.gsub!(@decimal_separator, ".") unless @decimal_separator.nil?
+          if value =~ @regexp
             case value
             when "NaN"
               return Float::NAN
@@ -184,7 +191,11 @@ module Csvlint
               when "‰"
                 return value.to_f / 1000
               else
-                return value.to_f
+                if @integer.nil?
+                  return value.include?(".") ? value.to_f : value.to_i
+                else
+                  return @integer ? value.to_i : value.to_f
+                end
               end
             end
           else
@@ -193,9 +204,13 @@ module Csvlint
         else
           match = @regexp.match(value)
           return nil if match.nil?
-          number = match["numeric_part"].gsub(@grouping_separator, "")
-          return number.to_i if @integer_pattern
-          return number.to_f
+          number = match["numeric_part"]
+          number.gsub!(@grouping_separator, "") unless @grouping_separator.nil?
+          number.gsub!(@decimal_separator, ".") unless @decimal_separator.nil?
+          number = @integer ? number.to_i : number.to_f
+          number = number.to_f / 100 if match["prefix"].include?("%") || match["suffix"].include?("%")
+          number = number.to_f / 1000 if match["prefix"].include?("‰") || match["suffix"].include?("‰")
+          return number
         end
       end
 

data/lib/csvlint/csvw/property_checker.rb

@@ -58,15 +58,29 @@ module Csvlint
                 else
                   if p[0] == "@"
                     raise Csvlint::Csvw::MetadataError.new(), "common property has property other than @id, @type, @value or @language beginning with @ (#{p})"
+                  else
+                    v, w = check_common_property_value(v, base_url, lang)
+                    warnings += Array(w)
                   end
                 end
-                if v.instance_of? Hash
-                  v, w = check_common_property_value(v, base_url, lang)
-                  warnings += Array(w)
-                end
                 value[p] = v
               end
               return value, warnings
+            when String
+              if lang == "und"
+                return value, nil
+              else
+                return { "@value" => value, "@language" => lang }, nil
+              end
+            when Array
+              values = []
+              warnings = []
+              value.each do |v|
+                v, w = check_common_property_value(v, base_url, lang)
+                warnings += Array(w)
+                values << v
+              end
+              return values, warnings
             else
               return value, nil
             end
@@ -114,6 +128,13 @@ module Csvlint
             }
           end
 
+          def uri_template_property(type)
+            return lambda { |value, base_url, lang|
+              return URITemplate.new(value), nil, type if value.instance_of? String
+              return URITemplate.new(""), :invalid_value, type
+            }
+          end
+
           def numeric_property(type)
             return lambda { |value, base_url, lang|
               return value, nil, type if value.kind_of?(Integer) && value >= 0
@@ -192,8 +213,43 @@ module Csvlint
           "@base" => link_property(:context),
           # common properties
           "@id" => link_property(:common),
-          "notes" => array_property(:common),
+          "notes" => lambda { |value, base_url, lang| 
+            return false, :invalid_value, :common unless value.instance_of? Array
+            values = []
+            warnings = []
+            value.each do |v|
+              v, w = check_common_property_value(v, base_url, lang)
+              values << v
+              warnings += w
+            end
+            return values, warnings, :common
+          },
           "suppressOutput" => boolean_property(:common),
+          "dialect" => lambda { |value, base_url, lang|
+            if value.instance_of? Hash
+              value = value.clone
+              warnings = []
+              value.each do |p,v|
+                if p == "@id"
+                  raise Csvlint::Csvw::MetadataError.new("dialect.@id"), "@id starts with _:" if v =~ /^_:/
+                elsif p == "@type"
+                  raise Csvlint::Csvw::MetadataError.new("dialect.@type"), "@type of dialect is not 'Dialect'" if v != 'Dialect'
+                else
+                  v, warning, type = check_property(p, v, base_url, lang)
+                  if type == :dialect && (warning.nil? || warning.empty?)
+                    value[p] = v
+                  else
+                    value.delete(p)
+                    warnings << :invalid_property unless type == :dialect
+                    warnings += Array(warning)
+                  end
+                end
+              end
+              return value, warnings, :common
+            else
+              return {}, :invalid_value, :common
+            end
+          },
           # inherited properties
           "null" => lambda { |value, base_url, lang|
             case value
@@ -270,12 +326,17 @@ module Csvlint
             warnings += convert_value_facet(value, "maxInclusive", value["base"])
             warnings += convert_value_facet(value, "maxExclusive", value["base"])
 
-            raise Csvlint::Csvw::MetadataError.new(""), "datatype cannot specify both minimum/minInclusive (#{value["minInclusive"]}) and minExclusive (#{value["minExclusive"]}" if value["minInclusive"] && value["minExclusive"]
-            raise Csvlint::Csvw::MetadataError.new(""), "datatype cannot specify both maximum/maxInclusive (#{value["maxInclusive"]}) and maxExclusive (#{value["maxExclusive"]}" if value["maxInclusive"] && value["maxExclusive"]
-            raise Csvlint::Csvw::MetadataError.new(""), "datatype minInclusive (#{value["minInclusive"]}) cannot be more than maxInclusive (#{value["maxInclusive"]}" if value["minInclusive"] && value["maxInclusive"] && value["minInclusive"] > value["maxInclusive"]
-            raise Csvlint::Csvw::MetadataError.new(""), "datatype minInclusive (#{value["minInclusive"]}) cannot be more than or equal to maxExclusive (#{value["maxExclusive"]}" if value["minInclusive"] && value["maxExclusive"] && value["minInclusive"] >= value["maxExclusive"]
-            raise Csvlint::Csvw::MetadataError.new(""), "datatype minExclusive (#{value["minExclusive"]}) cannot be more than or equal to maxExclusive (#{value["maxExclusive"]}" if value["minExclusive"] && value["maxExclusive"] && value["minExclusive"] > value["maxExclusive"]
-            raise Csvlint::Csvw::MetadataError.new(""), "datatype minExclusive (#{value["minExclusive"]}) cannot be more than maxInclusive (#{value["maxInclusive"]}" if value["minExclusive"] && value["maxInclusive"] && value["minExclusive"] >= value["maxInclusive"]
+            minInclusive = value["minInclusive"].is_a?(Hash) ? value["minInclusive"][:dateTime] : value["minInclusive"]
+            maxInclusive = value["maxInclusive"].is_a?(Hash) ? value["maxInclusive"][:dateTime] : value["maxInclusive"]
+            minExclusive = value["minExclusive"].is_a?(Hash) ? value["minExclusive"][:dateTime] : value["minExclusive"]
+            maxExclusive = value["maxExclusive"].is_a?(Hash) ? value["maxExclusive"][:dateTime] : value["maxExclusive"]
+
+            raise Csvlint::Csvw::MetadataError.new(""), "datatype cannot specify both minimum/minInclusive (#{minInclusive}) and minExclusive (#{minExclusive}" if minInclusive && minExclusive
+            raise Csvlint::Csvw::MetadataError.new(""), "datatype cannot specify both maximum/maxInclusive (#{maxInclusive}) and maxExclusive (#{maxExclusive}" if maxInclusive && maxExclusive
+            raise Csvlint::Csvw::MetadataError.new(""), "datatype minInclusive (#{minInclusive}) cannot be more than maxInclusive (#{maxInclusive}" if minInclusive && maxInclusive && minInclusive > maxInclusive
+            raise Csvlint::Csvw::MetadataError.new(""), "datatype minInclusive (#{minInclusive}) cannot be more than or equal to maxExclusive (#{maxExclusive}" if minInclusive && maxExclusive && minInclusive >= maxExclusive
+            raise Csvlint::Csvw::MetadataError.new(""), "datatype minExclusive (#{minExclusive}) cannot be more than or equal to maxExclusive (#{maxExclusive}" if minExclusive && maxExclusive && minExclusive > maxExclusive
+            raise Csvlint::Csvw::MetadataError.new(""), "datatype minExclusive (#{minExclusive}) cannot be more than maxInclusive (#{maxInclusive}" if minExclusive && maxInclusive && minExclusive >= maxInclusive
 
             raise Csvlint::Csvw::MetadataError.new(""), "datatype length (#{value["length"]}) cannot be less than minLength (#{value["minLength"]}" if value["length"] && value["minLength"] && value["length"] < value["minLength"]
             raise Csvlint::Csvw::MetadataError.new(""), "datatype length (#{value["length"]}) cannot be more than maxLength (#{value["maxLength"]}" if value["length"] && value["maxLength"] && value["length"] > value["maxLength"]
@@ -292,9 +353,9 @@ module Csvlint
               elsif NUMERIC_FORMAT_DATATYPES.include?(value["base"])
                 value["format"] = { "pattern" => value["format"] } if value["format"].instance_of? String
                 begin
-                  value["format"] = Csvlint::Csvw::NumberFormat.new(value["format"]["pattern"], value["format"]["groupChar"], value["format"]["decimalChar"] || ".")
+                  value["format"] = Csvlint::Csvw::NumberFormat.new(value["format"]["pattern"], value["format"]["groupChar"], value["format"]["decimalChar"] || ".", INTEGER_FORMAT_DATATYPES.include?(value["base"]))
                 rescue Csvlint::Csvw::NumberFormatError
-                  value["format"] = Csvlint::Csvw::NumberFormat.new(nil, value["format"]["groupChar"], value["format"]["decimalChar"] || ".")
+                  value["format"] = Csvlint::Csvw::NumberFormat.new(nil, value["format"]["groupChar"], value["format"]["decimalChar"] || ".", INTEGER_FORMAT_DATATYPES.include?(value["base"]))
                   warnings << :invalid_number_format
                 end
               elsif value["base"] == "http://www.w3.org/2001/XMLSchema#boolean"
@@ -326,9 +387,9 @@ module Csvlint
           },
           "required" => boolean_property(:inherited),
           "ordered" => boolean_property(:inherited),
-          "aboutUrl" => string_property(:inherited),
-          "propertyUrl" => string_property(:inherited),
-          "valueUrl" => string_property(:inherited),
+          "aboutUrl" => uri_template_property(:inherited),
+          "propertyUrl" => uri_template_property(:inherited),
+          "valueUrl" => uri_template_property(:inherited),
           "textDirection" => lambda { |value, base_url, lang|
             value = value.to_sym
             return value, nil, :inherited if [:ltr, :rtl, :auto, :inherit].include? value
@@ -421,31 +482,6 @@ module Csvlint
             return schema, warnings, :table
           },
           "url" => link_property(:table),
-          "dialect" => lambda { |value, base_url, lang|
-            if value.instance_of? Hash
-              value = value.clone
-              warnings = []
-              value.each do |p,v|
-                if p == "@id"
-                  raise Csvlint::Csvw::MetadataError.new("dialect.@id"), "@id starts with _:" if v =~ /^_:/
-                elsif p == "@type"
-                  raise Csvlint::Csvw::MetadataError.new("dialect.@type"), "@type of dialect is not 'Dialect'" if v != 'Dialect'
-                else
-                  v, warning, type = check_property(p, v, base_url, lang)
-                  if type == :dialect && (warning.nil? || warning.empty?)
-                    value[p] = v
-                  else
-                    value.delete(p)
-                    warnings << :invalid_property unless type == :dialect
-                    warnings += Array(warning)
-                  end
-                end
-              end
-              return value, warnings, :table
-            else
-              return {}, :invalid_value, :table
-            end
-          },
           # dialect properties
           "commentPrefix" => string_property(:dialect),
           "delimiter" => string_property(:dialect),
@@ -566,6 +602,12 @@ module Csvlint
           "xhv" => "http://www.w3.org/1999/xhtml/vocab#",
           "xml" => "http://www.w3.org/XML/1998/namespace",
           "xsd" => "http://www.w3.org/2001/XMLSchema#",
+          "csvw" => "http://www.w3.org/ns/csvw#",
+          "cnt" => "http://www.w3.org/2008/content",
+          "earl" => "http://www.w3.org/ns/earl#",
+          "ht" => "http://www.w3.org/2006/http#",
+          "oa" => "http://www.w3.org/ns/oa#",
+          "ptr" => "http://www.w3.org/2009/pointers#",
           "cc" => "http://creativecommons.org/ns#",
           "ctag" => "http://commontag.org/ns#",
           "dc" => "http://purl.org/dc/terms/",
@@ -638,8 +680,7 @@ module Csvlint
           "http://www.w3.org/2001/XMLSchema#hexBinary"
         ]
 
-        NUMERIC_FORMAT_DATATYPES = [
-          "http://www.w3.org/2001/XMLSchema#decimal",
+        INTEGER_FORMAT_DATATYPES = [
           "http://www.w3.org/2001/XMLSchema#integer",
           "http://www.w3.org/2001/XMLSchema#long",
           "http://www.w3.org/2001/XMLSchema#int",
@@ -652,10 +693,14 @@ module Csvlint
           "http://www.w3.org/2001/XMLSchema#unsignedShort",
           "http://www.w3.org/2001/XMLSchema#unsignedByte",
           "http://www.w3.org/2001/XMLSchema#nonPositiveInteger",
-          "http://www.w3.org/2001/XMLSchema#negativeInteger",
+          "http://www.w3.org/2001/XMLSchema#negativeInteger"
+        ]
+
+        NUMERIC_FORMAT_DATATYPES = [
+          "http://www.w3.org/2001/XMLSchema#decimal",
           "http://www.w3.org/2001/XMLSchema#double",
           "http://www.w3.org/2001/XMLSchema#float"
-        ]
+        ] + INTEGER_FORMAT_DATATYPES
 
         DATE_FORMAT_DATATYPES = [
           "http://www.w3.org/2001/XMLSchema#date",

data/lib/csvlint/csvw/table.rb

@@ -4,9 +4,9 @@ module Csvlint
 
       include Csvlint::ErrorCollector
 
-      attr_reader :columns, :dialect, :table_direction, :foreign_keys, :foreign_key_references, :id, :notes, :primary_key, :schema, :suppress_output, :transformations, :url, :annotations
+      attr_reader :columns, :dialect, :table_direction, :foreign_keys, :foreign_key_references, :id, :notes, :primary_key, :row_title_columns, :schema, :suppress_output, :transformations, :url, :annotations
 
-      def initialize(url, columns: [], dialect: {}, table_direction: :auto, foreign_keys: [], id: nil, notes: [], primary_key: nil, schema: nil, suppress_output: false, transformations: [], annotations: [], warnings: [])
+      def initialize(url, columns: [], dialect: {}, table_direction: :auto, foreign_keys: [], id: nil, notes: [], primary_key: nil, row_title_columns: [], schema: nil, suppress_output: false, transformations: [], annotations: [], warnings: [])
         @url = url
         @columns = columns
         @dialect = dialect
@@ -19,6 +19,7 @@ module Csvlint
         @notes = notes
         @primary_key = primary_key
         @primary_key_values = {}
+        @row_title_columns = row_title_columns
         @schema = schema
         @suppress_output = suppress_output
         @transformations = transformations
@@ -29,54 +30,59 @@ module Csvlint
         @warnings += columns.map{|c| c.warnings}.flatten
       end
 
-      def validate_header(headers)
+      def validate_header(headers, strict)
         reset
         headers.each_with_index do |header,i|
           if columns[i]
-            columns[i].validate_header(header)
+            columns[i].validate_header(header, strict)
             @errors += columns[i].errors
             @warnings += columns[i].warnings
-          else
+          elsif strict
             build_errors(:malformed_header, :schema, 1, nil, header, nil)
+          else
+            build_warnings(:malformed_header, :schema, 1, nil, header, nil)
           end
-        end unless columns.empty?
+        end # unless columns.empty?
         return valid?
       end
 
-      def validate_row(values, row=nil)
+      def validate_row(values, row=nil, validate=false)
         reset
         values.each_with_index do |value,i|
           column = columns[i]
           if column
-            column.validate(value, row)
+            v = column.validate(value, row)
+            values[i] = v
             @errors += column.errors
             @warnings += column.warnings
           else
             build_errors(:too_many_values, :schema, row, nil, value, nil)
           end
         end unless columns.empty?
-        unless @primary_key.nil?
-          key = @primary_key.map { |column| column.parse(values[column.number - 1], row) }
-          build_errors(:duplicate_key, :schema, row, nil, key.join(","), @primary_key_values[key]) if @primary_key_values.include?(key)
-          @primary_key_values[key] = row
-        end
-        # build a record of the unique values that are referenced by foreign keys from other tables
-        # so that later we can check whether those foreign keys reference these values
-        @foreign_key_references.each do |foreign_key|
-          referenced_columns = foreign_key["referenced_columns"]
-          key = referenced_columns.map{ |column| column.parse(values[column.number - 1], row) }
-          known_values = @foreign_key_reference_values[foreign_key] = @foreign_key_reference_values[foreign_key] || {}
-          known_values[key] = known_values[key] || []
-          known_values[key] << row
-        end
-        # build a record of the references from this row to other tables
-        # we can't check yet whether these exist in the other tables because
-        # we might not have parsed those other tables
-        @foreign_keys.each do |foreign_key|
-          referencing_columns = foreign_key["referencing_columns"]
-          key = referencing_columns.map{ |column| column.parse(values[column.number - 1], row) }
-          known_values = @foreign_key_values[foreign_key] = @foreign_key_values[foreign_key] || []
-          known_values << key unless known_values.include?(key)
+        if validate
+          unless @primary_key.nil?
+            key = @primary_key.map { |column| column.validate(values[column.number - 1], row) }
+            build_errors(:duplicate_key, :schema, row, nil, key.join(","), @primary_key_values[key]) if @primary_key_values.include?(key)
+            @primary_key_values[key] = row
+          end
+          # build a record of the unique values that are referenced by foreign keys from other tables
+          # so that later we can check whether those foreign keys reference these values
+          @foreign_key_references.each do |foreign_key|
+            referenced_columns = foreign_key["referenced_columns"]
+            key = referenced_columns.map{ |column| column.validate(values[column.number - 1], row) }
+            known_values = @foreign_key_reference_values[foreign_key] = @foreign_key_reference_values[foreign_key] || {}
+            known_values[key] = known_values[key] || []
+            known_values[key] << row
+          end
+          # build a record of the references from this row to other tables
+          # we can't check yet whether these exist in the other tables because
+          # we might not have parsed those other tables
+          @foreign_keys.each do |foreign_key|
+            referencing_columns = foreign_key["referencing_columns"]
+            key = referencing_columns.map{ |column| column.validate(values[column.number - 1], row) }
+            known_values = @foreign_key_values[foreign_key] = @foreign_key_values[foreign_key] || []
+            known_values << key unless known_values.include?(key)
+          end
         end
         return valid?
       end
@@ -107,19 +113,16 @@ module Csvlint
         return valid?
       end
 
-      def self.from_json(table_desc, base_url=nil, lang="und", inherited_properties={})
+      def self.from_json(table_desc, base_url=nil, lang="und", common_properties={}, inherited_properties={})
         annotations = {}
         warnings = []
-        table_properties = {}
         columns = []
-        notes = []
+        table_properties = common_properties.clone
         inherited_properties = inherited_properties.clone
 
         table_desc.each do |property,value|
           if property =="@type"
             raise Csvlint::Csvw::MetadataError.new("$.tables[?(@.url = '#{table_desc["url"]}')].@type"), "@type of table is not 'Table'" unless value == 'Table'
-          elsif property == "notes"
-            notes = value
           else
             v, warning, type = Csvw::PropertyChecker.check_property(property, value, base_url, lang)
             warnings += Array(warning).map{ |w| Csvlint::ErrorMessage.new(w, :metadata, nil, nil, "#{property}: #{value}", nil) } unless warning.nil? || warning.empty?
@@ -140,7 +143,17 @@ module Csvlint
         foreign_keys = []
         primary_key = nil
         if table_schema
-          raise Csvlint::Csvw::MetadataError.new("$.tables[?(@.url = '#{table_desc["url"]}')].tableSchema.columns"), "schema columns is not an array" unless table_schema["columns"].instance_of? Array
+          unless table_schema["columns"].instance_of? Array
+            table_schema["columns"] = []
+            warnings << Csvlint::ErrorMessage.new(:invalid_value, :metadata, nil, nil, "columns", nil)
+          end
+
+          table_schema.each do |p,v|
+            unless ["columns", "primaryKey", "foreignKeys", "rowTitles"].include? p
+              inherited_properties[p] = v
+            end
+          end
+
           virtual_columns = false
           table_schema["columns"].each_with_index do |column_desc,i|
             if column_desc.instance_of? Hash
@@ -180,8 +193,11 @@ module Csvlint
           end if foreign_keys
 
           row_titles = table_schema["rowTitles"]
-          row_titles.each_with_index do |row_title,i|
-            raise Csvlint::Csvw::MetadataError.new("$.tables[?(@.url = '#{table_desc["url"]}')].tableSchema.rowTitles[#{i}]"), "rowTitles references non-existant column" unless column_names.include? row_title
+          row_title_columns = []
+          row_titles.each_with_index do |row_title|
+            i = column_names.index(row_title)
+            raise Csvlint::Csvw::MetadataError.new("$.tables[?(@.url = '#{table_desc["url"]}')].tableSchema.rowTitles[#{i}]"), "rowTitles references non-existant column" unless i
+            row_title_columns << columns[i]
           end if row_titles
 
         end
@@ -191,9 +207,11 @@ module Csvlint
           columns: columns,
           dialect: table_properties["dialect"],
           foreign_keys: foreign_keys || [],
-          notes: notes,
+          notes: table_properties["notes"] || [],
           primary_key: primary_key_valid && !primary_key_columns.empty? ? primary_key_columns : nil,
+          row_title_columns: row_title_columns,
           schema: table_schema ? table_schema["@id"] : nil,
+          suppress_output: table_properties["suppressOutput"] ? table_properties["suppressOutput"] : false,
           annotations: annotations,
           warnings: warnings
         )