wjordan213.csvlint 0.2.8

data/lib/csvlint/csvw/table.rb

@@ -0,0 +1,204 @@
+module Csvlint
+  module Csvw
+    class Table
+
+      include Csvlint::ErrorCollector
+
+      attr_reader :columns, :dialect, :table_direction, :foreign_keys, :foreign_key_references, :id, :notes, :primary_key, :schema, :suppress_output, :transformations, :url, :annotations
+
+      def initialize(url, columns: [], dialect: {}, table_direction: :auto, foreign_keys: [], id: nil, notes: [], primary_key: nil, schema: nil, suppress_output: false, transformations: [], annotations: [], warnings: [])
+        @url = url
+        @columns = columns
+        @dialect = dialect
+        @table_direction = table_direction
+        @foreign_keys = foreign_keys
+        @foreign_key_values = {}
+        @foreign_key_references = []
+        @foreign_key_reference_values = {}
+        @id = id
+        @notes = notes
+        @primary_key = primary_key
+        @primary_key_values = {}
+        @schema = schema
+        @suppress_output = suppress_output
+        @transformations = transformations
+        @annotations = annotations
+        reset
+        @warnings += warnings
+        @errors += columns.map{|c| c.errors}.flatten
+        @warnings += columns.map{|c| c.warnings}.flatten
+      end
+
+      def validate_header(headers)
+        reset
+        headers.each_with_index do |header,i|
+          if columns[i]
+            columns[i].validate_header(header)
+            @errors += columns[i].errors
+            @warnings += columns[i].warnings
+          else
+            build_errors(:malformed_header, :schema, 1, nil, header, nil)
+          end
+        end unless columns.empty?
+        return valid?
+      end
+
+      def validate_row(values, row=nil)
+        reset
+        values.each_with_index do |value,i|
+          column = columns[i]
+          if column
+            column.validate(value, row)
+            @errors += column.errors
+            @warnings += column.warnings
+          else
+            build_errors(:too_many_values, :schema, row, nil, value, nil)
+          end
+        end unless columns.empty?
+        unless @primary_key.nil?
+          key = @primary_key.map { |column| column.parse(values[column.number - 1], row) }
+          build_errors(:duplicate_key, :schema, row, nil, key.join(","), @primary_key_values[key]) if @primary_key_values.include?(key)
+          @primary_key_values[key] = row
+        end
+        # build a record of the unique values that are referenced by foreign keys from other tables
+        # so that later we can check whether those foreign keys reference these values
+        @foreign_key_references.each do |foreign_key|
+          referenced_columns = foreign_key["referenced_columns"]
+          key = referenced_columns.map{ |column| column.parse(values[column.number - 1], row) }
+          known_values = @foreign_key_reference_values[foreign_key] = @foreign_key_reference_values[foreign_key] || {}
+          known_values[key] = known_values[key] || []
+          known_values[key] << row
+        end
+        # build a record of the references from this row to other tables
+        # we can't check yet whether these exist in the other tables because
+        # we might not have parsed those other tables
+        @foreign_keys.each do |foreign_key|
+          referencing_columns = foreign_key["referencing_columns"]
+          key = referencing_columns.map{ |column| column.parse(values[column.number - 1], row) }
+          known_values = @foreign_key_values[foreign_key] = @foreign_key_values[foreign_key] || []
+          known_values << key unless known_values.include?(key)
+        end
+        return valid?
+      end
+
+      def validate_foreign_keys
+        reset
+        @foreign_keys.each do |foreign_key|
+          local = @foreign_key_values[foreign_key]
+          remote_table = foreign_key["referenced_table"]
+          remote_table.validate_foreign_key_references(foreign_key, @url, local)
+          @errors += remote_table.errors unless remote_table == self
+          @warnings += remote_table.warnings unless remote_table == self
+        end
+        return valid?
+      end
+
+      def validate_foreign_key_references(foreign_key, remote_url, remote)
+        reset
+        local = @foreign_key_reference_values[foreign_key]
+        context = { "from" => { "url" => remote_url.to_s.split("/")[-1], "columns" => foreign_key["columnReference"] }, "to" => { "url" => @url.to_s.split("/")[-1], "columns" => foreign_key["reference"]["columnReference"] }}
+        remote.each do |r|
+          if local[r]
+            build_errors(:multiple_matched_rows, :schema, nil, nil, r, context) if local[r].length > 1
+          else
+            build_errors(:unmatched_foreign_key_reference, :schema, nil, nil, r, context)
+          end
+        end
+        return valid?
+      end
+
+      def self.from_json(table_desc, base_url=nil, lang="und", inherited_properties={})
+        annotations = {}
+        warnings = []
+        table_properties = {}
+        columns = []
+        notes = []
+        inherited_properties = inherited_properties.clone
+
+        table_desc.each do |property,value|
+          if property =="@type"
+            raise Csvlint::Csvw::MetadataError.new("$.tables[?(@.url = '#{table_desc["url"]}')].@type"), "@type of table is not 'Table'" unless value == 'Table'
+          elsif property == "notes"
+            notes = value
+          else
+            v, warning, type = Csvw::PropertyChecker.check_property(property, value, base_url, lang)
+            warnings += Array(warning).map{ |w| Csvlint::ErrorMessage.new(w, :metadata, nil, nil, "#{property}: #{value}", nil) } unless warning.nil? || warning.empty?
+            if type == :annotation
+              annotations[property] = v
+            elsif type == :table || type == :common
+              table_properties[property] = v
+            elsif type == :column
+              warnings << Csvlint::ErrorMessage.new(:invalid_property, :metadata, nil, nil, "#{property}", nil)
+            else
+              inherited_properties[property] = v
+            end
+          end
+        end
+
+        table_schema = table_properties["tableSchema"] || inherited_properties["tableSchema"]
+        column_names = []
+        foreign_keys = []
+        primary_key = nil
+        if table_schema
+          raise Csvlint::Csvw::MetadataError.new("$.tables[?(@.url = '#{table_desc["url"]}')].tableSchema.columns"), "schema columns is not an array" unless table_schema["columns"].instance_of? Array
+          virtual_columns = false
+          table_schema["columns"].each_with_index do |column_desc,i|
+            if column_desc.instance_of? Hash
+              column = Csvlint::Csvw::Column.from_json(i+1, column_desc, base_url, lang, inherited_properties)
+              raise Csvlint::Csvw::MetadataError.new("$.tables[?(@.url = '#{table_desc["url"]}')].tableSchema.columns[#{i}].virtual"), "virtual columns before non-virtual column #{column.name || i}" if virtual_columns && !column.virtual
+              virtual_columns = virtual_columns || column.virtual
+              raise Csvlint::Csvw::MetadataError.new("$.tables[?(@.url = '#{table_desc["url"]}')].tableSchema.columns"), "multiple columns named #{column.name}" if column_names.include? column.name
+              column_names << column.name unless column.name.nil?
+              columns << column
+            else
+              warnings << Csvlint::ErrorMessage.new(:invalid_column_description, :metadata, nil, nil, "#{column_desc}", nil)
+            end
+          end
+
+          primary_key = table_schema["primaryKey"]
+          primary_key_columns = []
+          primary_key_valid = true
+          primary_key.each do |reference|
+            i = column_names.index(reference)
+            if i
+              primary_key_columns << columns[i]
+            else
+              warnings << Csvlint::ErrorMessage.new(:invalid_column_reference, :metadata, nil, nil, "primaryKey: #{reference}", nil)
+              primary_key_valid = false
+            end
+          end if primary_key
+
+          foreign_keys = table_schema["foreignKeys"]
+          foreign_keys.each_with_index do |foreign_key, i|
+            foreign_key_columns = []
+            foreign_key["columnReference"].each do |reference|
+              i = column_names.index(reference)
+              raise Csvlint::Csvw::MetadataError.new("$.tables[?(@.url = '#{table_desc["url"]}')].tableSchema.foreignKeys[#{i}].columnReference"), "foreignKey references non-existant column" unless i
+              foreign_key_columns << columns[i]
+            end
+            foreign_key["referencing_columns"] = foreign_key_columns
+          end if foreign_keys
+
+          row_titles = table_schema["rowTitles"]
+          row_titles.each_with_index do |row_title,i|
+            raise Csvlint::Csvw::MetadataError.new("$.tables[?(@.url = '#{table_desc["url"]}')].tableSchema.rowTitles[#{i}]"), "rowTitles references non-existant column" unless column_names.include? row_title
+          end if row_titles
+
+        end
+
+        return self.new(table_properties["url"],
+          id: table_properties["@id"],
+          columns: columns,
+          dialect: table_properties["dialect"],
+          foreign_keys: foreign_keys || [],
+          notes: notes,
+          primary_key: primary_key_valid && !primary_key_columns.empty? ? primary_key_columns : nil,
+          schema: table_schema ? table_schema["@id"] : nil,
+          annotations: annotations,
+          warnings: warnings
+        )
+      end
+
+    end
+  end
+end

data/lib/csvlint/csvw/table_group.rb

@@ -0,0 +1,165 @@
+module Csvlint
+  module Csvw
+    class TableGroup
+
+      include Csvlint::ErrorCollector
+
+      attr_reader :url, :id, :tables, :notes, :annotations
+
+      def initialize(url, id: nil, tables: {}, notes: [], annotations: {}, warnings: [])
+        @url = url
+        @id = id
+        @tables = tables
+        @notes = notes
+        @annotations = annotations
+        @validated_tables = {}
+        @tables.each { |t,v| @validated_tables[t] = false }
+        reset
+        @warnings += warnings
+        @errors += @tables.map{|url,table| table.errors}.flatten
+        @warnings += @tables.map{|url,table| table.warnings}.flatten
+      end
+
+      def validate_header(header, table_url)
+        reset
+        table_url = "file:#{File.absolute_path(table_url)}" if table_url.instance_of? File
+        table = tables[table_url]
+        table.validate_header(header)
+        @errors += table.errors
+        @warnings += table.warnings
+        return valid?
+      end
+
+      def validate_row(values, row=nil, all_errors=[], table_url)
+        reset
+        table_url = "file:#{File.absolute_path(table_url)}" if table_url.instance_of? File
+        @validated_tables[table_url] = true
+        table = tables[table_url]
+        table.validate_row(values, row)
+        @errors += table.errors
+        @warnings += table.warnings
+        return valid?
+      end
+
+      def validate_foreign_keys
+        reset
+        unless @validated_tables.has_value?(false)
+          @tables.each do |table_url,table|
+            table.validate_foreign_keys
+            @errors += table.errors
+            @warnings += table.warnings
+          end
+        end
+        return valid?
+      end
+
+      def self.from_json(url, json)
+        warnings = []
+        tables = {}
+        annotations = {}
+        inherited_properties = {}
+        common_properties = {}
+        base_url = URI(url.to_s.strip)
+        lang = "und"
+
+        context = json["@context"]
+        if context.instance_of?(Array) && context[1]
+          context[1].each do |property,value|
+            v, warning, type = Csvw::PropertyChecker.check_property(property, value, base_url, lang)
+            if warning.nil? || warning.empty?
+              if type == :context
+                base_url = v if property == "@base"
+                lang = v if property == "@language"
+              else
+                raise Csvlint::Csvw::MetadataError.new("$.@context"), "@context contains properties other than @base or @language (#{property})"
+              end
+            else
+              raise Csvlint::Csvw::MetadataError.new("$.@context"), "@context contains properties other than @base or @language (#{property})" unless ["@base", "@language"].include?(property)
+              warnings += Array(warning).map{ |w| Csvlint::ErrorMessage.new(w, :metadata, nil, nil, "@context: #{property}: #{value}", nil) }
+            end
+          end
+        end
+        json.delete("@context")
+
+        if json["url"]
+          json = { "tables" => [ json ] }
+        end unless json["tables"]
+
+        json.each do |property,value|
+          unless VALID_PROPERTIES.include? property
+            v, warning, type = Csvw::PropertyChecker.check_property(property, value, base_url, lang)
+            warnings += Array(warning).map{ |w| Csvlint::ErrorMessage.new(w, :metadata, nil, nil, "#{property}: #{value}", nil) } unless warning.nil? || warning.empty?
+            if type == :annotation
+              annotations[property] = v
+            elsif type == :common
+              common_properties[property] = v
+            elsif type == :column
+              warnings << Csvlint::ErrorMessage.new(:invalid_property, :metadata, nil, nil, "#{property}", nil)
+            else
+              inherited_properties[property] = v
+            end
+          end
+        end
+
+        id = common_properties["@id"]
+
+        raise Csvlint::Csvw::MetadataError.new("$.@type"), "@type of table group is not 'TableGroup'" if json["@type"] && json["@type"] != 'TableGroup'
+
+        raise Csvlint::Csvw::MetadataError.new("$"), "no tables property" unless json["tables"]
+        raise Csvlint::Csvw::MetadataError.new("$.tables"), "empty tables property" if json["tables"].empty?
+        raise Csvlint::Csvw::MetadataError.new("$.tables"), "tables property is not an array" unless json["tables"].instance_of? Array
+
+        json["tables"].each do |table_desc|
+          if table_desc.instance_of? Hash
+            table_url = table_desc["url"]
+            unless table_url.instance_of? String
+              warnings << Csvlint::ErrorMessage.new(:invalid_url, :metadata, nil, nil, "url: #{table_url}", nil)
+              table_url = ""
+            end
+            table_url = URI.join(base_url, table_url).to_s
+            table_desc["url"] = table_url
+            table = Csvlint::Csvw::Table.from_json(table_desc, base_url, lang, inherited_properties)
+            tables[table_url] = table
+          else
+            warnings << Csvlint::ErrorMessage.new(:invalid_table_description, :metadata, nil, nil, "#{table_desc}", nil)
+          end
+        end
+
+        tables.each do |table_url, table|
+          table.foreign_keys.each_with_index do |foreign_key,i|
+            reference = foreign_key["reference"]
+            if reference["resource"]
+              resource = URI.join(base_url, reference["resource"]).to_s
+              referenced_table = tables[resource]
+              raise Csvlint::Csvw::MetadataError.new("$.tables[?(@.url = '#{table_url}')].tableSchema.foreign_keys[#{i}].reference.resource"), "foreign key references table that does not exist (#{resource})" if referenced_table.nil?
+            else
+              schema_url = URI.join(base_url, reference["schemaReference"]).to_s
+              referenced_tables = tables.values.select{ |table| table.schema == schema_url }
+              referenced_table = referenced_tables[0]
+              raise Csvlint::Csvw::MetadataError.new("$.tables[?(@.url = '#{table_url}')].tableSchema.foreign_keys[#{i}].reference.schemaReference"), "foreign key references schema that is not used (#{schema_url})" if referenced_table.nil?
+            end
+            foreign_key["referenced_table"] = referenced_table
+            table_columns = {}
+            referenced_table.columns.each do |column|
+              table_columns[column.name] = column if column.name
+            end
+            referenced_columns = []
+            Array(reference["columnReference"]).each do |column_reference|
+              column = table_columns[column_reference]
+              raise Csvlint::Csvw::MetadataError.new("$.tables[?(@.url = '#{table_url}')].tableSchema.foreign_keys[#{i}].reference.columnReference"), "column named #{column_reference} does not exist in #{resource}" if column.nil?
+              referenced_columns << column
+            end
+            foreign_key["referenced_columns"] = referenced_columns
+            referenced_table.foreign_key_references << foreign_key
+          end
+        end
+
+        return self.new(base_url, id: id, tables: tables, notes: json["notes"] || [], annotations: annotations, warnings: warnings)
+      end
+
+      private
+        VALID_PROPERTIES = ['tables', 'notes', '@type']
+
+    end
+  end
+end

data/lib/csvlint/error_collector.rb

@@ -0,0 +1,27 @@
+module Csvlint
+  module ErrorCollector
+    attr_reader :errors, :warnings, :info_messages
+    # Creates a validation error
+    def build_errors(type, category = nil, row = nil, column = nil, content = nil, constraints = {})
+      @errors << Csvlint::ErrorMessage.new(type, category, row, column, content, constraints)
+    end
+    # Creates a validation warning
+    def build_warnings(type, category = nil, row = nil, column = nil, content = nil, constraints = {})
+      @warnings << Csvlint::ErrorMessage.new(type, category, row, column, content, constraints)
+    end
+    # Creates a validation information message
+    def build_info_messages(type, category = nil, row = nil, column = nil, content = nil, constraints = {})
+      @info_messages << Csvlint::ErrorMessage.new(type, category, row, column, content, constraints)
+    end
+
+    def valid?
+      errors.empty?
+    end
+
+    def reset
+      @errors = []
+      @warnings = []
+      @info_messages = []
+    end
+  end
+end

data/lib/csvlint/error_message.rb

@@ -0,0 +1,15 @@
+module Csvlint
+  class ErrorMessage
+    attr_reader :type, :category, :row, :column, :content, :constraints
+
+    def initialize(type, category, row, column, content, constraints)
+      @type = type
+      @category = category
+      @row = row
+      @column = column
+      @content = content
+      @constraints = constraints
+
+    end
+  end
+end

data/lib/csvlint/field.rb

@@ -0,0 +1,196 @@
+module Csvlint
+
+  class Field
+    include Csvlint::ErrorCollector
+
+    attr_reader :name, :constraints, :title, :description
+
+    def initialize(name, constraints={}, title=nil, description=nil)
+      @name = name
+      @constraints = constraints || {}
+      @uniques = Set.new
+      @title = title
+      @description = description
+      reset
+    end
+
+    def validate_column(value, row=nil, column=nil, all_errors=[])
+      reset
+      unless all_errors.any?{|error| ((error.type == :invalid_regex) && (error.column == column))}
+        validate_regex(value, row, column, all_errors)
+      end
+      validate_length(value, row, column)
+      validate_values(value, row, column)
+      parsed = validate_type(value, row, column)
+      validate_range(parsed, row, column) if parsed != nil
+      return valid?
+    end
+
+    private
+      def validate_length(value, row, column)
+        if constraints["required"] == true
+          build_errors(:missing_value, :schema, row, column, value,
+            { "required" => true }) if value.nil? || value.length == 0
+        end
+        if constraints["minLength"]
+          build_errors(:min_length, :schema, row, column, value,
+            { "minLength" => constraints["minLength"] }) if value.nil? || value.length < constraints["minLength"]
+        end
+        if constraints["maxLength"]
+            build_errors(:max_length, :schema, row, column, value,
+             { "maxLength" => constraints["maxLength"] } ) if !value.nil? && value.length > constraints["maxLength"]
+        end
+      end
+
+      def validate_regex(value, row, column, all_errors)
+        pattern = constraints["pattern"]
+        if pattern
+          begin
+            Regexp.new(pattern)
+            build_errors(:pattern, :schema, row, column, value,
+            { "pattern" => constraints["pattern"] } ) if !value.nil? && !value.match( constraints["pattern"] )
+          rescue RegexpError
+            build_regex_error(value, row, column, pattern, all_errors)
+          end
+        end
+      end
+
+      def build_regex_error(value, row, column, pattern, all_errors)
+        return if @regex_error_exists
+        build_errors(:invalid_regex, :schema, nil, column, ("#{name}: Constraints: Pattern: #{pattern}"),
+          { "pattern" => constraints["pattern"] })
+        @regex_error_exists = true
+      end
+
+      def validate_values(value, row, column)
+        # If a pattern exists, raise an invalid regex error if it is not in
+        # valid regex form, else, if the value of the relevant field in the csv
+        # does not match the given regex pattern in the schema, raise a
+        # pattern error.
+        if constraints["unique"] == true
+          if @uniques.include? value
+            build_errors(:unique, :schema, row, column, value, { "unique" => true })
+          else
+            @uniques << value
+          end
+        end
+      end
+
+      def validate_type(value, row, column)
+        if constraints["type"] && value != ""
+          parsed = convert_to_type(value)
+          if parsed == nil
+            failed = { "type" => constraints["type"] }
+            failed["datePattern"] = constraints["datePattern"] if constraints["datePattern"]
+            build_errors(:invalid_type, :schema, row, column, value, failed)
+            return nil
+          end
+          return parsed
+        end
+        return nil
+      end
+
+      def validate_range(value, row, column)
+        #TODO: we're ignoring issues with converting ranges to actual types, maybe we
+        #should generate a warning? The schema is invalid
+        if constraints["minimum"]
+          minimumValue = convert_to_type( constraints["minimum"] )
+          if minimumValue
+            build_errors(:below_minimum, :schema, row, column, value,
+              { "minimum" => constraints["minimum"] }) unless value >= minimumValue
+          end
+        end
+        if constraints["maximum"]
+          maximumValue = convert_to_type( constraints["maximum"] )
+          if maximumValue
+            build_errors(:above_maximum, :schema, row, column, value,
+            { "maximum" => constraints["maximum"] }) unless value <= maximumValue
+          end
+        end
+      end
+
+      def convert_to_type(value)
+        parsed = nil
+        tv = TYPE_VALIDATIONS[constraints["type"]]
+        if tv
+          begin
+            parsed = tv.call value, constraints
+          rescue ArgumentError
+          end
+        end
+        return parsed
+      end
+
+      TYPE_VALIDATIONS = {
+          'http://www.w3.org/2001/XMLSchema#string'  => lambda { |value, constraints| value },
+          'http://www.w3.org/2001/XMLSchema#int'     => lambda { |value, constraints| Integer value },
+          'http://www.w3.org/2001/XMLSchema#integer' => lambda { |value, constraints| Integer value },
+          'http://www.w3.org/2001/XMLSchema#float'   => lambda { |value, constraints| Float value },
+          'http://www.w3.org/2001/XMLSchema#double'   => lambda { |value, constraints| Float value },
+          'http://www.w3.org/2001/XMLSchema#anyURI'  => lambda do |value, constraints|
+            begin
+              u = URI.parse value
+              raise ArgumentError unless u.kind_of?(URI::HTTP) || u.kind_of?(URI::HTTPS)
+            rescue URI::InvalidURIError
+              raise ArgumentError
+            end
+            u
+          end,
+          'http://www.w3.org/2001/XMLSchema#boolean' => lambda do |value, constraints|
+            return true if ['true', '1'].include? value
+            return false if ['false', '0'].include? value
+            raise ArgumentError
+          end,
+          'http://www.w3.org/2001/XMLSchema#nonPositiveInteger' => lambda do |value, constraints|
+            i = Integer value
+            raise ArgumentError unless i <= 0
+            i
+          end,
+          'http://www.w3.org/2001/XMLSchema#negativeInteger' => lambda do |value, constraints|
+            i = Integer value
+            raise ArgumentError unless i < 0
+            i
+          end,
+          'http://www.w3.org/2001/XMLSchema#nonNegativeInteger' => lambda do |value, constraints|
+            i = Integer value
+            raise ArgumentError unless i >= 0
+            i
+          end,
+          'http://www.w3.org/2001/XMLSchema#positiveInteger' => lambda do |value, constraints|
+            i = Integer value
+            raise ArgumentError unless i > 0
+            i
+          end,
+          'http://www.w3.org/2001/XMLSchema#dateTime' => lambda do |value, constraints|
+            date_pattern = constraints["datePattern"] || "%Y-%m-%dT%H:%M:%SZ"
+            d = DateTime.strptime(value, date_pattern)
+            raise ArgumentError unless d.strftime(date_pattern) == value
+            d
+          end,
+          'http://www.w3.org/2001/XMLSchema#date' => lambda do |value, constraints|
+            date_pattern = constraints["datePattern"] || "%Y-%m-%d"
+            d = Date.strptime(value, date_pattern)
+            raise ArgumentError unless d.strftime(date_pattern) == value
+            d
+          end,
+          'http://www.w3.org/2001/XMLSchema#time' => lambda do |value, constraints|
+            date_pattern = constraints["datePattern"] || "%H:%M:%S"
+            d = DateTime.strptime(value, date_pattern)
+            raise ArgumentError unless d.strftime(date_pattern) == value
+            d
+          end,
+          'http://www.w3.org/2001/XMLSchema#gYear' => lambda do |value, constraints|
+            date_pattern = constraints["datePattern"] || "%Y"
+            d = Date.strptime(value, date_pattern)
+            raise ArgumentError unless d.strftime(date_pattern) == value
+            d
+          end,
+          'http://www.w3.org/2001/XMLSchema#gYearMonth' => lambda do |value, constraints|
+            date_pattern = constraints["datePattern"] || "%Y-%m"
+            d = Date.strptime(value, date_pattern)
+            raise ArgumentError unless d.strftime(date_pattern) == value
+            d
+          end,
+      }
+  end
+end