csvlint 0.1.4 → 0.2.0

data/lib/csvlint/csvw/table.rb

@@ -0,0 +1,204 @@
+module Csvlint
+  module Csvw
+    class Table
+
+      include Csvlint::ErrorCollector
+
+      attr_reader :columns, :dialect, :table_direction, :foreign_keys, :foreign_key_references, :id, :notes, :primary_key, :schema, :suppress_output, :transformations, :url, :annotations
+
+      def initialize(url, columns: [], dialect: {}, table_direction: :auto, foreign_keys: [], id: nil, notes: [], primary_key: nil, schema: nil, suppress_output: false, transformations: [], annotations: [], warnings: [])
+        @url = url
+        @columns = columns
+        @dialect = dialect
+        @table_direction = table_direction
+        @foreign_keys = foreign_keys
+        @foreign_key_values = {}
+        @foreign_key_references = []
+        @foreign_key_reference_values = {}
+        @id = id
+        @notes = notes
+        @primary_key = primary_key
+        @primary_key_values = {}
+        @schema = schema
+        @suppress_output = suppress_output
+        @transformations = transformations
+        @annotations = annotations
+        reset
+        @warnings += warnings
+        @errors += columns.map{|c| c.errors}.flatten
+        @warnings += columns.map{|c| c.warnings}.flatten
+      end
+
+      def validate_header(headers)
+        reset
+        headers.each_with_index do |header,i|
+          if columns[i]
+            columns[i].validate_header(header)
+            @errors += columns[i].errors
+            @warnings += columns[i].warnings
+          else
+            build_errors(:malformed_header, :schema, 1, nil, header, nil)
+          end
+        end unless columns.empty?
+        return valid?
+      end
+
+      def validate_row(values, row=nil)
+        reset
+        values.each_with_index do |value,i|
+          column = columns[i]
+          if column
+            column.validate(value, row)
+            @errors += column.errors
+            @warnings += column.warnings
+          else
+            build_errors(:too_many_values, :schema, row, nil, value, nil)
+          end
+        end unless columns.empty?
+        unless @primary_key.nil?
+          key = @primary_key.map { |column| column.parse(values[column.number - 1], row) }
+          build_errors(:duplicate_key, :schema, row, nil, key.join(","), @primary_key_values[key]) if @primary_key_values.include?(key)
+          @primary_key_values[key] = row
+        end
+        # build a record of the unique values that are referenced by foreign keys from other tables
+        # so that later we can check whether those foreign keys reference these values
+        @foreign_key_references.each do |foreign_key|
+          referenced_columns = foreign_key["referenced_columns"]
+          key = referenced_columns.map{ |column| column.parse(values[column.number - 1], row) }
+          known_values = @foreign_key_reference_values[foreign_key] = @foreign_key_reference_values[foreign_key] || {}
+          known_values[key] = known_values[key] || []
+          known_values[key] << row
+        end
+        # build a record of the references from this row to other tables
+        # we can't check yet whether these exist in the other tables because
+        # we might not have parsed those other tables
+        @foreign_keys.each do |foreign_key|
+          referencing_columns = foreign_key["referencing_columns"]
+          key = referencing_columns.map{ |column| column.parse(values[column.number - 1], row) }
+          known_values = @foreign_key_values[foreign_key] = @foreign_key_values[foreign_key] || []
+          known_values << key unless known_values.include?(key)
+        end
+        return valid?
+      end
+
+      def validate_foreign_keys
+        reset
+        @foreign_keys.each do |foreign_key|
+          local = @foreign_key_values[foreign_key]
+          remote_table = foreign_key["referenced_table"]
+          remote_table.validate_foreign_key_references(foreign_key, @url, local)
+          @errors += remote_table.errors unless remote_table == self
+          @warnings += remote_table.warnings unless remote_table == self
+        end
+        return valid?
+      end
+
+      def validate_foreign_key_references(foreign_key, remote_url, remote)
+        reset
+        local = @foreign_key_reference_values[foreign_key]
+        context = { "from" => { "url" => remote_url.to_s.split("/")[-1], "columns" => foreign_key["columnReference"] }, "to" => { "url" => @url.to_s.split("/")[-1], "columns" => foreign_key["reference"]["columnReference"] }}
+        remote.each do |r|
+          if local[r]
+            build_errors(:multiple_matched_rows, :schema, nil, nil, r, context) if local[r].length > 1
+          else
+            build_errors(:unmatched_foreign_key_reference, :schema, nil, nil, r, context)
+          end
+        end
+        return valid?
+      end
+
+      def self.from_json(table_desc, base_url=nil, lang="und", inherited_properties={})
+        annotations = {}
+        warnings = []
+        table_properties = {}
+        columns = []
+        notes = []
+        inherited_properties = inherited_properties.clone
+
+        table_desc.each do |property,value|
+          if property =="@type"
+            raise Csvlint::Csvw::MetadataError.new("$.tables[?(@.url = '#{table_desc["url"]}')].@type"), "@type of table is not 'Table'" unless value == 'Table'
+          elsif property == "notes"
+            notes = value
+          else
+            v, warning, type = Csvw::PropertyChecker.check_property(property, value, base_url, lang)
+            warnings += Array(warning).map{ |w| Csvlint::ErrorMessage.new(w, :metadata, nil, nil, "#{property}: #{value}", nil) } unless warning.nil? || warning.empty?
+            if type == :annotation
+              annotations[property] = v
+            elsif type == :table || type == :common
+              table_properties[property] = v
+            elsif type == :column
+              warnings << Csvlint::ErrorMessage.new(:invalid_property, :metadata, nil, nil, "#{property}", nil)
+            else
+              inherited_properties[property] = v
+            end
+          end
+        end
+
+        table_schema = table_properties["tableSchema"] || inherited_properties["tableSchema"]
+        column_names = []
+        foreign_keys = []
+        primary_key = nil
+        if table_schema
+          raise Csvlint::Csvw::MetadataError.new("$.tables[?(@.url = '#{table_desc["url"]}')].tableSchema.columns"), "schema columns is not an array" unless table_schema["columns"].instance_of? Array
+          virtual_columns = false
+          table_schema["columns"].each_with_index do |column_desc,i|
+            if column_desc.instance_of? Hash
+              column = Csvlint::Csvw::Column.from_json(i+1, column_desc, base_url, lang, inherited_properties)
+              raise Csvlint::Csvw::MetadataError.new("$.tables[?(@.url = '#{table_desc["url"]}')].tableSchema.columns[#{i}].virtual"), "virtual columns before non-virtual column #{column.name || i}" if virtual_columns && !column.virtual
+              virtual_columns = virtual_columns || column.virtual
+              raise Csvlint::Csvw::MetadataError.new("$.tables[?(@.url = '#{table_desc["url"]}')].tableSchema.columns"), "multiple columns named #{column.name}" if column_names.include? column.name
+              column_names << column.name unless column.name.nil?
+              columns << column
+            else
+              warnings << Csvlint::ErrorMessage.new(:invalid_column_description, :metadata, nil, nil, "#{column_desc}", nil)
+            end
+          end
+
+          primary_key = table_schema["primaryKey"]
+          primary_key_columns = []
+          primary_key_valid = true
+          primary_key.each do |reference|
+            i = column_names.index(reference)
+            if i
+              primary_key_columns << columns[i]
+            else
+              warnings << Csvlint::ErrorMessage.new(:invalid_column_reference, :metadata, nil, nil, "primaryKey: #{reference}", nil)
+              primary_key_valid = false
+            end
+          end if primary_key
+
+          foreign_keys = table_schema["foreignKeys"]
+          foreign_keys.each_with_index do |foreign_key, i|
+            foreign_key_columns = []
+            foreign_key["columnReference"].each do |reference|
+              i = column_names.index(reference)
+              raise Csvlint::Csvw::MetadataError.new("$.tables[?(@.url = '#{table_desc["url"]}')].tableSchema.foreignKeys[#{i}].columnReference"), "foreignKey references non-existant column" unless i
+              foreign_key_columns << columns[i]
+            end
+            foreign_key["referencing_columns"] = foreign_key_columns
+          end if foreign_keys
+
+          row_titles = table_schema["rowTitles"]
+          row_titles.each_with_index do |row_title,i|
+            raise Csvlint::Csvw::MetadataError.new("$.tables[?(@.url = '#{table_desc["url"]}')].tableSchema.rowTitles[#{i}]"), "rowTitles references non-existant column" unless column_names.include? row_title
+          end if row_titles
+
+        end
+
+        return self.new(table_properties["url"],
+          id: table_properties["@id"],
+          columns: columns,
+          dialect: table_properties["dialect"],
+          foreign_keys: foreign_keys || [],
+          notes: notes,
+          primary_key: primary_key_valid && !primary_key_columns.empty? ? primary_key_columns : nil,
+          schema: table_schema ? table_schema["@id"] : nil,
+          annotations: annotations,
+          warnings: warnings
+        )
+      end
+
+    end
+  end
+end

data/lib/csvlint/csvw/table_group.rb

@@ -0,0 +1,165 @@
+module Csvlint
+  module Csvw
+    class TableGroup
+
+      include Csvlint::ErrorCollector
+
+      attr_reader :url, :id, :tables, :notes, :annotations
+
+      def initialize(url, id: nil, tables: {}, notes: [], annotations: {}, warnings: [])
+        @url = url
+        @id = id
+        @tables = tables
+        @notes = notes
+        @annotations = annotations
+        @validated_tables = {}
+        @tables.each { |t,v| @validated_tables[t] = false }
+        reset
+        @warnings += warnings
+        @errors += @tables.map{|url,table| table.errors}.flatten
+        @warnings += @tables.map{|url,table| table.warnings}.flatten
+      end
+
+      def validate_header(header, table_url)
+        reset
+        table_url = "file:#{File.absolute_path(table_url)}" if table_url.instance_of? File
+        table = tables[table_url]
+        table.validate_header(header)
+        @errors += table.errors
+        @warnings += table.warnings
+        return valid?
+      end
+
+      def validate_row(values, row=nil, all_errors=[], table_url)
+        reset
+        table_url = "file:#{File.absolute_path(table_url)}" if table_url.instance_of? File
+        @validated_tables[table_url] = true
+        table = tables[table_url]
+        table.validate_row(values, row)
+        @errors += table.errors
+        @warnings += table.warnings
+        return valid?
+      end
+
+      def validate_foreign_keys
+        reset
+        unless @validated_tables.has_value?(false)
+          @tables.each do |table_url,table|
+            table.validate_foreign_keys
+            @errors += table.errors
+            @warnings += table.warnings
+          end
+        end
+        return valid?
+      end
+
+      def self.from_json(url, json)
+        warnings = []
+        tables = {}
+        annotations = {}
+        inherited_properties = {}
+        common_properties = {}
+        base_url = URI(url.to_s.strip)
+        lang = "und"
+
+        context = json["@context"]
+        if context.instance_of?(Array) && context[1]
+          context[1].each do |property,value|
+            v, warning, type = Csvw::PropertyChecker.check_property(property, value, base_url, lang)
+            if warning.nil? || warning.empty?
+              if type == :context
+                base_url = v if property == "@base"
+                lang = v if property == "@language"
+              else
+                raise Csvlint::Csvw::MetadataError.new("$.@context"), "@context contains properties other than @base or @language (#{property})"
+              end
+            else
+              raise Csvlint::Csvw::MetadataError.new("$.@context"), "@context contains properties other than @base or @language (#{property})" unless ["@base", "@language"].include?(property)
+              warnings += Array(warning).map{ |w| Csvlint::ErrorMessage.new(w, :metadata, nil, nil, "@context: #{property}: #{value}", nil) }
+            end
+          end
+        end
+        json.delete("@context")
+
+        if json["url"]
+          json = { "tables" => [ json ] }
+        end unless json["tables"]
+
+        json.each do |property,value|
+          unless VALID_PROPERTIES.include? property
+            v, warning, type = Csvw::PropertyChecker.check_property(property, value, base_url, lang)
+            warnings += Array(warning).map{ |w| Csvlint::ErrorMessage.new(w, :metadata, nil, nil, "#{property}: #{value}", nil) } unless warning.nil? || warning.empty?
+            if type == :annotation
+              annotations[property] = v
+            elsif type == :common
+              common_properties[property] = v
+            elsif type == :column
+              warnings << Csvlint::ErrorMessage.new(:invalid_property, :metadata, nil, nil, "#{property}", nil)
+            else
+              inherited_properties[property] = v
+            end
+          end
+        end
+
+        id = common_properties["@id"]
+
+        raise Csvlint::Csvw::MetadataError.new("$.@type"), "@type of table group is not 'TableGroup'" if json["@type"] && json["@type"] != 'TableGroup'
+
+        raise Csvlint::Csvw::MetadataError.new("$"), "no tables property" unless json["tables"]
+        raise Csvlint::Csvw::MetadataError.new("$.tables"), "empty tables property" if json["tables"].empty?
+        raise Csvlint::Csvw::MetadataError.new("$.tables"), "tables property is not an array" unless json["tables"].instance_of? Array
+
+        json["tables"].each do |table_desc|
+          if table_desc.instance_of? Hash
+            table_url = table_desc["url"]
+            unless table_url.instance_of? String
+              warnings << Csvlint::ErrorMessage.new(:invalid_url, :metadata, nil, nil, "url: #{table_url}", nil)
+              table_url = ""
+            end
+            table_url = URI.join(base_url, table_url).to_s
+            table_desc["url"] = table_url
+            table = Csvlint::Csvw::Table.from_json(table_desc, base_url, lang, inherited_properties)
+            tables[table_url] = table
+          else
+            warnings << Csvlint::ErrorMessage.new(:invalid_table_description, :metadata, nil, nil, "#{table_desc}", nil)
+          end
+        end
+
+        tables.each do |table_url, table|
+          table.foreign_keys.each_with_index do |foreign_key,i|
+            reference = foreign_key["reference"]
+            if reference["resource"]
+              resource = URI.join(base_url, reference["resource"]).to_s
+              referenced_table = tables[resource]
+              raise Csvlint::Csvw::MetadataError.new("$.tables[?(@.url = '#{table_url}')].tableSchema.foreign_keys[#{i}].reference.resource"), "foreign key references table that does not exist (#{resource})" if referenced_table.nil?
+            else
+              schema_url = URI.join(base_url, reference["schemaReference"]).to_s
+              referenced_tables = tables.values.select{ |table| table.schema == schema_url }
+              referenced_table = referenced_tables[0]
+              raise Csvlint::Csvw::MetadataError.new("$.tables[?(@.url = '#{table_url}')].tableSchema.foreign_keys[#{i}].reference.schemaReference"), "foreign key references schema that is not used (#{schema_url})" if referenced_table.nil?
+            end
+            foreign_key["referenced_table"] = referenced_table
+            table_columns = {}
+            referenced_table.columns.each do |column|
+              table_columns[column.name] = column if column.name
+            end
+            referenced_columns = []
+            Array(reference["columnReference"]).each do |column_reference|
+              column = table_columns[column_reference]
+              raise Csvlint::Csvw::MetadataError.new("$.tables[?(@.url = '#{table_url}')].tableSchema.foreign_keys[#{i}].reference.columnReference"), "column named #{column_reference} does not exist in #{resource}" if column.nil?
+              referenced_columns << column
+            end
+            foreign_key["referenced_columns"] = referenced_columns
+            referenced_table.foreign_key_references << foreign_key
+          end
+        end
+
+        return self.new(base_url, id: id, tables: tables, notes: json["notes"] || [], annotations: annotations, warnings: warnings)
+      end
+
+      private
+        VALID_PROPERTIES = ['tables', 'notes', '@type']
+
+    end
+  end
+end

data/lib/csvlint/schema.rb

@@ -14,7 +14,45 @@ module Csvlint
       reset
     end
 
-    def validate_header(header)
+    class << self
+
+      def from_json_table(uri, json)
+        fields = []
+        json["fields"].each do |field_desc|
+          fields << Csvlint::Field.new( field_desc["name"] , field_desc["constraints"],
+            field_desc["title"], field_desc["description"] )
+        end if json["fields"]
+        return Schema.new( uri , fields, json["title"], json["description"] )
+      end
+
+      def from_csvw_metadata(uri, json)
+        return Csvlint::Csvw::TableGroup.from_json(uri, json)
+      end
+
+      def load_from_json(uri)
+        begin
+          json = JSON.parse( open(uri).read )
+          if json["@context"]
+            uri = "file:#{File.expand_path(uri)}" unless uri.to_s =~ /^http(s)?/
+            return Schema.from_csvw_metadata(uri,json)
+          else
+            return Schema.from_json_table(uri,json)
+          end
+        rescue Csvlint::Csvw::MetadataError => e
+          raise e
+        rescue OpenURI::HTTPError => e
+          raise e
+        rescue => e
+          STDERR.puts e.class
+          STDERR.puts e.message
+          STDERR.puts e.backtrace
+          return Schema.new(nil, [], "malformed", "malformed")
+        end
+      end
+
+    end
+
+    def validate_header(header, source_url=nil)
       reset
 
       found_header = header.to_csv(:row_sep => '')
@@ -25,7 +63,7 @@ module Csvlint
       return valid?
     end
 
-    def validate_row(values, row=nil, all_errors=[])
+    def validate_row(values, row=nil, all_errors=[], source_url=nil)
       reset
       if values.length < fields.length
         fields[values.size..-1].each_with_index do |field, i|
@@ -48,26 +86,5 @@ module Csvlint
       return valid?
     end
 
-    def Schema.from_json_table(uri, json)
-      fields = []
-      json["fields"].each do |field_desc|
-        fields << Csvlint::Field.new( field_desc["name"] , field_desc["constraints"],
-          field_desc["title"], field_desc["description"] )
-      end if json["fields"]
-      return Schema.new( uri , fields, json["title"], json["description"] )
-    end
-
-    # Difference in functionality between from_json_table and load_from_json_table
-    # needs to be specified
-
-    def Schema.load_from_json_table(uri)
-      begin
-        json = JSON.parse( open(uri).read )
-        return Schema.from_json_table(uri,json)
-      rescue
-        return Schema.new(nil, [], "malformed", "malformed")
-      end
-    end
-
   end
 end