dwcr 0.0.8

data/lib/dwcr/metaschema/attribute.rb

@@ -0,0 +1,84 @@
+# frozen_string_literal: true
+
+#
+module DwCR
+  module Metaschema
+
+  # This class represents _field_ nodes in the DarwinCoreArchive
+  # that correspond to columns in table sof the DwCA schema
+  # and where applicable the ContentFile instances
+  # * +type+: the column type of the node
+  #   default: 'string'
+  # * +name+: the name for the node
+  #   default: term without namespace in underscore (snake case) form
+  # * +term+: the full term (a uri), including namespace for the node
+  #   see http://rs.tdwg.org/dwc/terms/index.htm
+  # * +default+: the default vale for columns in the
+  #   DwCA schema table
+  # * +index+: the column index in the ContentFile instances
+  #   associated with the Attribute instances' parent Entity instance
+  # * +max_content_length+: the maximum string length of values
+  #   in the corresponding column in the ContentFile instances
+  #   associated with the Attribute instances' parent Entity instance
+  # * *#entity*:
+  #   the Entity instance the Attribute instance belongs to
+  class Attribute < Sequel::Model
+    include XMLParsable
+
+    many_to_one :entity
+
+    # Returns the last component of the term
+    # will return the full term is the term is not unambiguous
+    def baseterm
+      unambiguous ? term&.split('/')&.last : term
+    end
+
+    # Returns a symbol for the +name+ that is
+    # the name of the column in the DarwinCoreArchive schema
+    def column_name
+      name.to_sym
+    end
+
+    # Reurns +true+ if the _field_ node is the foreign key in the
+    # _core_ or an _extension_ node in the DwCA schema, +false+ otherwise
+    def foreign_key?
+      index == entity.key_column && !entity.is_core
+    end
+
+    # Returns the maximum length for values in the corresponding column
+    # in the DwCA schema
+    # the returned value is the greater of either the length of the +default+
+    # or the +max_content_length+ or +nil+ if neither is set
+    def length
+      [default&.length, max_content_length].compact.max
+    end
+
+    # Returns an array that can be splatted as arguments into the
+    # Sequel::Schema::CreatTableGenerator#column method:
+    # <tt>[name, type, options]</tt>
+    def to_table_column
+      col_type = type ? type.to_sym : :string
+      [column_name, col_type, { index: index_options, default: default }]
+    end
+
+    alias length= max_content_length=
+
+    private
+
+    # Returns the index options for the Sequel database column
+    # by default DwCR does not persist the foreign key field of any _extension_
+    # nodes in the DwCA schema (as that relation is re-established through
+    # associations) using proper SQL foreign keys
+    # therefore the below else clause will never be reached in DwCR files
+    # generated from an existing DarwinCoreArchive
+    def index_options
+      return false unless index && index == entity.key_column
+      if entity.is_core
+        { unique: true }
+      else
+        true
+      end
+    end
+  end
+  end
+end

data/lib/dwcr/metaschema/content_file.rb

@@ -0,0 +1,130 @@
+# frozen_string_literal: true
+
+#
+module DwCR
+  #
+  module Metaschema
+    # This class represents _location_ nodes (children of the _files_ nodes)
+    # in DarwinCoreArchive, which represent csv files associated with a
+    # _core_ or _extension_ node
+    # * +name+:
+    #   the basename of the file with extension (normally .csv)
+    # * +path+:
+    #   the directory path where the file is located
+    #   set this attribute to load content files from arbitrary directories
+    # * +is_loaded+:
+    #   a flag that is set to +true+ when the ContentFile instance's
+    #   contents have been loaded from the CSV file into the database
+    # * *#entity*:
+    #   the Entity instance the ContentFile instance belongs to
+    class ContentFile < Sequel::Model
+      many_to_one :entity
+
+      # Returns the full file name including the path
+      def file_name
+        File.join(path, name)
+      end
+
+      # Returns an array of symbols for column names for each column in the
+      # (headerless) CSV file specified in the +file+ attribute
+      # the array is mapped from the ContentFile instance's parent Entity
+      # instance's Attribute instances that have an +index+
+      # The array is sorted by the +index+
+      def content_headers
+        entity.attributes_dataset
+              .exclude(index: nil)
+              .exclude(type: nil)
+              .order(:index)
+              .map(&:column_name)
+      end
+
+      # Inserts all rows of the CSV file belonging to the ContentFile instance
+      # into the table of the DwCA represented by the instance's parent
+      # Entity instance
+      # Will raise an error for _extension_ instances
+      # if the _core_ instance is not loaded
+      def load
+        return if is_loaded
+        load_error = 'core needs to be loaded before extension files'
+        raise load_error unless entity.is_core || entity.core.loaded?
+        CSV.foreach(file_name) { |row| insert_row(row) }
+        self.is_loaded = true
+        save
+        is_loaded
+      end
+
+      # Deletes all rows of the CSV file belonging to the ContentFile instance
+      # from the table of the DwCA represented by the instance's parent
+      # Entity instance
+      # *Warning*: If this is called on a _core_ instance,
+      # this will also destroy any dependant _extension_ records!
+      def unload!
+        return unless is_loaded
+        CSV.foreach(file_name) { |row| delete_row(row) }
+        self.is_loaded = false
+        save
+      end
+
+      private
+
+      # removes all cells from a row for which the column in all
+      # associated CSV files is empty (the associated Attribute instance
+      # has <tt>type == nil</tt>)
+      def compact(row)
+        empty_cols = entity.attributes_dataset
+                           .exclude(index: nil)
+                           .where(type: nil)
+                           .map(&:index)
+        row.delete_if.with_index { |_, index| empty_cols.include?(index) }
+      end
+
+      # Delets a CSV row from the DwCA table represented
+      # by the instance's parent Entity instance
+      def delete_row(row)
+        row_vals = row_to_hash row
+        row_vals.delete(entity.key) unless entity.is_core
+        rec = entity.model_get.first(row_vals)
+        rec.destroy
+      end
+
+      # Inserts a CSV row into the DwCA schema's _core_ table
+      def insert_into_core(row)
+        return unless entity.is_core
+        entity.model_get.create(row_to_hash(row))
+      end
+
+      # Inserts a CSV row into an _extension_ table of the DwCA schema
+      def insert_into_extension(row)
+        row_vals = row_to_hash row
+        core_row(row_vals.delete(entity.key)).send(add_related, row_vals)
+      end
+
+      # Inserts a CSV row into the DwCA table represented
+      # by the instance's parent Entity instance
+      def insert_row(row)
+        rec = insert_into_core(row) || insert_into_extension(row)
+        entity.send(add_related, rec)
+      end
+
+      # Returns the row from the DwCA _core_ that matches the foreign key
+      def core_row(foreign_key)
+        entity.core.model_get.first(entity.core.key => foreign_key)
+      end
+
+      # Returns a string that is the method name to add a related record via
+      # an association on the DwCA model
+      def add_related
+        'add_' + entity.name.singularize
+      end
+
+      # Creates a hash from a headerless CSV row
+      # Entity instance's :attributes colum names are keys
+      # the CSV row cells are values
+      def row_to_hash(row)
+        keys = content_headers
+        vals = compact row
+        keys.zip(vals).to_h.select { |_k, v| v && !v.empty? }
+      end
+    end
+  end
+end

data/lib/dwcr/metaschema/entity.rb

@@ -0,0 +1,195 @@
+# frozen_string_literal: true
+
+#
+module DwCR
+  #
+  module Metaschema
+    # This class represents _core_ or _extension_ nodes in DarwinCoreArchive
+    # * +name+: the name for the node
+    #   default: pluralized term without namespace
+    #   in underscore (snake case) form
+    # * +term+: the full term (a uri), including namespace for the node
+    #   see http://rs.tdwg.org/dwc/terms/index.htm
+    # * +is_core+:
+    #   _true_ if the node is the core node of the DarwinCoreArchive,
+    #   _false_ otherwise
+    # * +key_column+: the column in the node representing
+    #   the primary key of the core node or
+    #   the foreign key in am extension node
+    # * +fields_enclosed_by+: directive to form CSV in :content_files
+    #   default: '"'
+    # * +fields_terminated_by+: fieldsTerminatedBy=","
+    # * +lines_terminated_by+: linesTerminatedBy="\r\n"
+    # * *#archive*:
+    #   the Archive instance the Entity instance belongs to
+    # * *#attributes*:
+    #   Attribute instances associated with the Entity
+    # * *#content_files*:
+    #   ContentFile instances associated with the Entity
+    # * *#core*:
+    #   if the Entity instance is an _extension_
+    #   returns the Entity instance representing the _core_ node
+    #   nil otherwise
+    # * *#extensions*:
+    #   if the Entity instance is the _core_ node
+    #   returns any Entity instances representing _extension_ nodes
+    class Entity < Sequel::Model
+      include XMLParsable
+
+      ensure_not_core = lambda do |ent, attr|
+        e = 'extensions must be associated with a core'
+        attr.is_core = false
+        raise ArgumentError, e unless ent.is_core
+        attr.archive = ent.archive
+      end
+
+      ensure_path = lambda do |ent, attr|
+        attr.path ||= ent.archive.path
+      end
+
+      ensure_unique_name = lambda do |ent, attr|
+        attr.name ||= attr.term&.split('/')&.last&.underscore
+        name_taken = ent.attributes_dataset.first(name: attr.name)
+        if name_taken
+          attr.name += '!'
+          attr.unambiguous = false
+          name_taken.unambiguous = false
+          name_taken.save
+        end
+      end
+
+      many_to_one :archive
+      one_to_many :attributes, before_add: ensure_unique_name
+      one_to_many :content_files, before_add: ensure_path
+      many_to_one :core, class: self
+      one_to_many :extensions, key: :core_id, class: self,
+                               before_add: ensure_not_core
+
+      # Returns the last component of the term
+      def baseterm
+        term.split('/').last
+      end
+
+      # Returns a string with Entity instance's singularized name in camelcase
+      # this is the name of the Sequel::Model in the DarwinCoreArchive schema
+      def class_name
+        name.classify
+      end
+
+      # Returns an array of full filenames with path
+      # for all associated ContentFile instances
+      def files
+        content_files.map(&:file_name)
+      end
+
+      # Returns *true* if all _content_files_ have been loaded,
+      # _false_ otherwise
+      def loaded?
+        loaded_files = content_files_dataset.where(is_loaded: true)
+        return true if loaded_files.count == content_files.size
+        loaded_files.empty? ? false : loaded_files.map(&:file_name)
+      end
+
+      # Returns a symbol based on the Entity instance's foreign key name
+      def foreign_key
+        class_name.foreign_key.to_sym
+      end
+
+      # Returns a symbol for the +name+ of
+      # the associated Attribute instance that is the key_column
+      # in the DarwinCoreArchive node the Entity represents
+      def key
+        attributes_dataset.first(index: key_column).name.to_sym
+      end
+
+      # Returns an array of parameters for all associations of the Sequel::Model
+      # in the DarwinCoreArchive schema that the Entity represents
+      # each set of parameters is an array
+      # <tt>[association_type, association_name, options]</tt>
+      # that can be splattet as arguments into
+      # Sequel::Model::Associations::ClassMethods#associate
+      def model_associations
+        # add the assoc to Entity here
+        meta_assoc = [:many_to_one, :entity, { class: Entity }]
+        if is_core
+          a = extensions.map { |extension| association_with(extension) }
+          a.unshift meta_assoc
+        else
+          [meta_assoc, association_with(core)]
+        end
+      end
+
+      # Returns the constant with module name for the Entity instance
+      # this is the constant of the Sequel::Model
+      # in the DarwinCoreArchive schema
+      def model_get
+        modelname = 'DwCR::' + class_name
+        modelname.constantize
+      end
+
+      # Returns a symbol for the pluralzid +name+ that is
+      # the name of the table in the DarwinCoreArchive schema
+      def table_name
+        name.tableize.to_sym
+      end
+
+      # Analyzes the Entity instance's content_files
+      # for any parameters given as modifiers and updates the
+      # asssociated _attributes_ with the new values
+      # * _:length_ or _'length'_
+      #   will update the Attribute instances' +max_content_length+
+      # * _:type_ or _'type'_
+      #   will update the Attribute instances' +type+ attributes
+      def update_attributes!(*modifiers)
+        DwCAContentAnalyzer::FileSet.new(files, modifiers).columns.each do |cp|
+          column = attributes_dataset.first(index: cp[:index])
+          modifiers.each { |m| column.send(m.to_s + '=', cp[m]) }
+          column.save
+        end
+      end
+
+      # Methods to add records to the :attributes and
+      # :content_files associations form xml
+
+      # Creates a Attribute instance from an xml node (_field_)
+      # given that the instance has not been previously defined
+      # if an instance has been previously defined, it will be updated
+      def add_attribute_from(xml)
+        attribute = attributes_dataset.first(term: term_from(xml))
+        attribute ||= add_attribute(values_from(xml, :term, :index, :default))
+        attribute.update_from(xml, :index, :default)
+      end
+
+      # Creates a ContentFile instance from an xml node (_file_)
+      # a +path+ can be given to add files from arbitrary directories
+      # +name+ is parsed from the _location_ node
+      def add_files_from(xml, path: nil)
+        files_from(xml).each { |file| add_content_file(name: file, path: path) }
+      end
+
+      private
+
+      # Sequel Model hook that creates a default +name+ from the +term+
+      def before_create
+        e = 'Entity instances need to belong to a Archive'
+        raise ArgumentError, e unless archive
+        self.name ||= term&.split('/')&.last&.underscore
+        super
+      end
+
+      # Returns an array that can be splattet as arguments into the
+      # Sequel::Model::Associations::ClassMethods#associate method:
+      # <tt>[association_type, association_name, options]</tt>
+      def association_with(entity)
+        options = { class: entity.class_name, class_namespace: 'DwCR' }
+        if is_core
+          options[:key] = foreign_key
+          [:one_to_many, entity.table_name, options]
+        else
+          options[:key] = entity.foreign_key
+          [:many_to_one, entity.name.singularize.to_sym, options]
+        end
+      end
+    end
+  end
+end

data/lib/dwcr/metaschema/metaschema.rb

@@ -0,0 +1,94 @@
+# frozen_string_literal: true
+
+require_relative 'xml_parsable'
+
+#
+module DwCR
+  # This module provides functionality to create, update, and verify the
+  # part of the DwCR schema needed to persist information about the DwCR
+  module Metaschema
+    # Creates the tables for the metaschema present in every DwCR
+    # _archives_, _entities_, _attributes_, _content_files_
+    # loads the Sequel::Model classes for these tables
+    def self.create
+      tabledefs = Psych.load_file(File.join(__dir__, 'metaschema_tables.yml'))
+      tabledefs.to_h.each do |table, columns|
+        DB.create_table? table do
+          primary_key :id
+          columns.each { |c| column(*c) }
+        end
+      end
+      load_models
+    end
+
+    # Loads all Sequel::Model classes for the metaschema
+    def self.load_models
+      require_relative 'archive'
+      require_relative 'entity'
+      require_relative 'attribute'
+      require_relative 'content_file'
+    end
+
+    # Returns schema or index parameters for a table, depending on the
+    # second argument (+:schema+ or +:indexes+)
+    # returns +false+ if the table does not exist
+    def self.inspect_table(table, method)
+      DB.indexes(table).values.map { |x| x[:columns] }.flatten
+      DB.send(method, table)
+    rescue Sequel::Error
+      false
+    end
+
+    # Performs an integrety check on +table+, veryfies all columns
+    # are present with the parameters given in +columns+;
+    # a column parameter is an array with the structure:
+    # <tt>[:column_name, :column_type, {column_options} ]</tt>
+    def self.columns?(table, *columns)
+      db_cols = inspect_table(table, :schema)
+      return unless db_cols
+      exp_cols = columns.map(&:first).unshift(:id)
+      exp_cols == db_cols.map(&:first)
+    end
+
+    # Performs an integrety check on +table+, veryfies all indices
+    # are present with the parameters given in +columns+;
+    # a column parameter is an array with the structure:
+    # <tt>[:column_name, :column_type, {column_options} ]</tt>
+    def self.indexes?(table, *columns)
+      db_idxs = inspect_table(table, :indexes)
+      return unless db_idxs
+      exp_idxs = columns.select { |column| column[2]&.fetch(:index, false) }
+                        .map(&:first)
+      exp_idxs & db_idxs.values.map { |x| x[:columns] }.flatten == exp_idxs
+    end
+
+    # Updates all Attribute instances in a Archive
+    # with parameters from files in ContentFile
+    # _schema_options_: a Hash with attribute names as keys and boolean values
+    # <tt>{ :type => true, :length => true }</tt>
+    # updates any attribute given as key where value is _true_
+    def self.update(archive, **options)
+      return if options.empty?
+
+      # FIXME: handle situation where schema tables have been created
+      options.select! { |_k, v| v == true }
+      archive.entities
+             .each { |entity| entity.update_attributes!(*options.keys) }
+    end
+
+    # Performs an integrity check on the metaschema in the DWCR file
+    # (the current database connection)
+    # returns true if all tables, columns, and indices as given in
+    # _config/metaschema_tables.yml_ are present
+    def self.valid?
+      tabledefs = Psych.load_file('lib/dwcr/metaschema/metaschema_tables.yml')
+      status = tabledefs.map do |td|
+        table = td.first
+        columns = td.last
+        columns?(table, *columns) && indexes?(table, *columns)
+      end
+      return false if status.uniq.size > 1
+      status.first
+    end
+  end
+end

data/lib/dwcr/metaschema/metaschema_tables.yml

@@ -0,0 +1,130 @@
+---
+!ruby/sym archives:
+  -
+    - !ruby/sym core_id
+    - !ruby/sym integer
+    - !ruby/sym index:
+        true
+  -
+    - !ruby/sym name
+    - !ruby/sym string
+  -
+    - !ruby/sym path
+    - !ruby/sym string
+  -
+    - !ruby/sym xmlns
+    - !ruby/sym string
+    - !ruby/sym default:
+        'http://rs.tdwg.org/dwc/text/'
+  -
+    - !ruby/sym xmlns__xs
+    - !ruby/sym string
+    - !ruby/sym default:
+        'http://www.w3.org/2001/XMLSchema'
+  -
+    - !ruby/sym xmlns__xsi
+    - !ruby/sym string
+    - !ruby/sym default:
+        'http://www.w3.org/2001/XMLSchema-instance'
+  -
+    - !ruby/sym xsi__schema_location
+    - !ruby/sym string
+    - !ruby/sym default:
+        'http://rs.tdwg.org/dwc/text/ http://rs.tdwg.org/dwc/text/tdwg_dwc_text.xsd'
+!ruby/sym attributes:
+  -
+    - !ruby/sym entity_id
+    - !ruby/sym integer
+    - !ruby/sym index:
+        true
+  -
+    - !ruby/sym type
+    - !ruby/sym string
+    - !ruby/sym default:
+        string
+  -
+    - !ruby/sym name
+    - !ruby/sym string
+    -
+      !ruby/sym index: true
+      !ruby/sym null: false
+  -
+    - !ruby/sym term
+    - !ruby/sym string
+  -
+    - !ruby/sym default
+    - !ruby/sym string
+  -
+    - !ruby/sym index
+    - !ruby/sym integer
+  -
+    - !ruby/sym max_content_length
+    - !ruby/sym integer
+  -
+    - !ruby/sym unambiguous
+    - !ruby/sym boolean
+    - !ruby/sym default:
+        true
+
+
+!ruby/sym entities:
+  -
+    - !ruby/sym archive_id
+    - !ruby/sym integer
+    - !ruby/sym index:
+        true
+  -
+    - !ruby/sym core_id
+    - !ruby/sym integer
+    - !ruby/sym index:
+        true
+  -
+    - !ruby/sym name
+    - !ruby/sym string
+    - !ruby/sym null:
+        false
+  -
+    - !ruby/sym term
+    - !ruby/sym string
+  -
+    - !ruby/sym is_core
+    - !ruby/sym boolean
+  -
+    - !ruby/sym key_column
+    - !ruby/sym integer
+  -
+    - !ruby/sym fields_enclosed_by
+    - !ruby/sym string
+    - !ruby/sym default:
+        '"'
+  -
+    - !ruby/sym fields_terminated_by
+    - !ruby/sym string
+    - !ruby/sym default:
+        ','
+  -
+    - !ruby/sym lines_terminated_by
+    - !ruby/sym string
+    - !ruby/sym default:
+        '\r\n'
+
+!ruby/sym content_files:
+  -
+    - !ruby/sym entity_id
+    - !ruby/sym integer
+    - !ruby/sym index:
+        true
+  -
+    - !ruby/sym name
+    - !ruby/sym string
+    - !ruby/sym null:
+        false
+  -
+    - !ruby/sym path
+    - !ruby/sym string
+  -
+    - !ruby/sym is_loaded
+    - !ruby/sym boolean
+    - !ruby/sym default:
+        false
+