dwcr 0.0.8

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: e1c757829c2f62acf12a5ca5c7eef5a5473d60ee1ef40500adc5b3df34ca0296
+  data.tar.gz: f58cb0ee2250939b93ef081b3f7a0ab7d02ed02c1f426b854f91df9a0c8864be
+SHA512:
+  metadata.gz: da0050e27a07dd812f5e0421f86681c7b6d24d2c2a8e31d8d844af163e1cd3bccde3dd53af11d86a1a0b8defe26f6e648fb88e0aa080a3492e93206ea6144f95
+  data.tar.gz: 5b17dc2bff6c355766edd300b04c40cdda3cbe3089cfd69cafaeca90a3965ca2342710cc4a7379c7107ce0984956a176070ca6691742d3348fb83fe7cd092b0c

data/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2017 Martin Stein
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

data/README.md

@@ -0,0 +1,24 @@
+# DwCR
+DwCA in a SQLite database complete with
+[Sequel](https://github.com/jeremyevans/sequel) models.
+
+## Command Line Tool
+
+### Create a new DwCR file
+
+From within the directory of a DwC Archive:
+
+```ruby
+dwcr new -t dwcr-test.db
+```
+
+will create the file in the directory, named after the directory
+
+see `dwcr new --help` for options
+
+### Load an existing DwCR file
+
+```ruby
+dwcr load ~/documents/dwca/dwcr-test.db
+```
+

data/bin/dwcr

@@ -0,0 +1,32 @@
+#!/usr/bin/env ruby
+
+# frozen_string_literal: true
+
+require 'getoptlong'
+require 'pry'
+require 'dwcr'
+require 'cli/shell'
+
+# require_relative '../lib/dwcr'
+# require_relative '../lib/cli/shell'
+
+dwcr_cmds = %w[load new]
+
+this_cmd = dwcr_cmds.include?(ARGV[0]) ? ARGV.shift : nil
+
+SHELL = CLI::Shell.new this_cmd
+
+if this_cmd
+  scrpt_dir = __dir__.split('/')
+  scrpt_dir.pop
+  scrpt_dir.concat %W[lib cli #{this_cmd}.rb]
+  cmd_scrpt = File.join scrpt_dir
+  load cmd_scrpt
+end
+
+SHELL.options.each do |opt, arg|
+  case opt
+  when '--help'
+    SHELL.print_help
+  end
+end

data/lib/cli/help.yml

@@ -0,0 +1,66 @@
+---
+NULL:
+  usage: >
+    dwcr [command] [options]
+  options:
+    -
+      - '--help'
+      - '-h'
+      - NO_ARGUMENT
+      - show help
+load:
+  usage: |
+    dwcr load [file] [options]
+    loads an existing DwCR file
+  options:
+    -
+      - '--help'
+      - '-h'
+      - NO_ARGUMENT
+      - show help
+    -
+      - '--interactive'
+      - '-i'
+      - NO_ARGUMENT
+      - |
+        starts an interactive Pry session
+new:
+  usage: |
+    dwcr new [options]
+    creates a new DwCR
+    default name for the DwCR is the
+  options:
+    -
+      - '--help'
+      - '-h'
+      - NO_ARGUMENT
+      - show help
+    -
+      - '--coltypes'
+      - '-c'
+      - NO_ARGUMENT
+      - |
+        set the column types based on contents from the included csv files
+        by default all columns will be of type 'text'
+    -
+      - '--meta'
+      - '-m'
+      - REQUIRED_ARGUMENT
+      - |
+        specify a meta.xml file for the schema
+        by default will search for meta.xml in the working directory
+        (see --path option)
+    -
+      - '--path'
+      - '-p'
+      - REQUIRED_ARGUMENT
+      - |
+        the directory where the DarwinCore Archive is located
+        defaults to the current directory
+    -
+      - '--target'
+      - '-t'
+      - OPTIONAL_ARGUMENT
+      - |
+        the target file to be created; if no argument is provided
+        this option will create an in-memory the SQLite database

data/lib/cli/load.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module CLI
+  SHELL.options.each do |opt, arg|
+    case opt
+    when '--help'
+      SHELL.print_help
+      exit true
+    when '--interactive'
+      SHELL.session = true
+    end
+  end
+
+  SHELL.target = ARGV.shift
+
+  ::DB = Sequel.sqlite(SHELL.target)
+
+  DwCR::Metaschema.load_models
+
+  DwCR::MODELS = DwCR.load_models
+
+  binding.pry if SHELL.session
+
+  exit true
+end

data/lib/cli/new.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+module CLI
+  schema_opts = {}
+
+  SHELL.options.each do |opt, arg|
+    case opt
+    when '--help'
+      SHELL.print_help
+      exit true
+    when '--coltypes'
+      schema_opts[:type] = true
+    when '--meta'
+      xml = arg
+    when '--path'
+      SHELL.path = arg
+    when '--target'
+      SHELL.target = arg.empty? ? nil : arg
+    end
+  end
+
+  xml ||= SHELL.path
+
+  ::DB = Sequel.sqlite(SHELL.target)
+
+  DwCR::Metaschema.create
+
+  archive = DwCR::Metaschema::Archive.create(path: SHELL.path)
+
+  meta_doc = DwCR::Metaschema::XMLParsable.load_meta xml
+
+  archive.load_nodes_from meta_doc
+
+  DwCR.create_schema(archive, schema_opts)
+
+  DwCR::MODELS = DwCR.load_models archive
+
+  DwCR.load_contents_for archive
+
+  binding.pry
+
+  puts 'done!'
+end

data/lib/cli/shell.rb

@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+require 'getoptlong'
+require 'psych'
+
+#
+module CLI
+  #
+  class Shell
+    attr_accessor :path, :session
+    attr_reader :options, :target
+
+    def initialize(cmd)
+      cmd_shell = Psych.load_file(File.join(__dir__, 'help.yml'))[cmd]
+      @usage = ["Usage: #{cmd_shell['usage']}\n"]
+      @options = nil
+      @usage << load_options(cmd_shell['options'])
+      @path = Dir.pwd
+      @target = target_directory @path
+      @session = false
+    end
+
+    def print_help
+      puts @usage
+    end
+
+    def target=(target_path)
+      @target = target_path ? target_directory(target_path) || target_path : nil
+    end
+
+    private
+
+    def load_options(raw_opts)
+      cmd_opts = []
+      pp_opts = raw_opts.map do |opt|
+        cmd_opts << opt[0..1].append(GetoptLong.const_get(opt[2]))
+        str = "    #{opt[1]}    #{opt[0]}"
+        fill = ''
+        (40 - str.length).times { fill += ' ' }
+        blankfill = ''
+        40.times { blankfill += ' ' }
+        optlines = opt[3].lines
+        firstline = str + fill + optlines.shift
+        [firstline, optlines.map { |line| blankfill + line }].join#("\n")
+      end
+      @options = GetoptLong.new(*cmd_opts)
+      pp_opts
+    end
+
+    def target_directory(target_path)
+      return nil unless File.directory? target_path
+      File.join(target_path, File.basename(@path) + '.db')
+    end
+  end
+end

data/lib/dwca_content_analyzer/column.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+#
+module DwCAContentAnalyzer
+  #
+  class Column
+    attr_reader :index, :type, :length
+
+    def initialize(index, contents, *detectors)
+      raise ArgumentError unless index.is_a? Integer
+      detectors = [] if detectors.include? :none
+      detectors = %i[type= length=] if detectors.include? :all
+      detectors.map! { |d| (d.id2name + '=').to_sym }
+      @index = index
+      @type = nil
+      @length = nil
+      analyze(contents, detectors)
+    end
+
+    private
+
+    def analyze(contents, detectors)
+      return if detectors.empty?
+      cells = contents.compact
+      detectors.each { |detector| send(detector, cells) }
+    end
+
+    # collapses all types encountered in a file's column into a single type
+    def collapse(types)
+      return types.first if types.size == 1
+      return nil if types.empty?
+      return String if string?(types)
+      return Float if float?(types)
+      String
+    end
+
+    def length=(cells)
+      @length = cells.map(&:to_s).map(&:length).max || 0
+    end
+
+    def type=(cells)
+      @type = collapse(cells.map(&:class).uniq)
+    end
+
+    def float?(types)
+      types.size == 2 && types.include?(Float) && types.include?(Integer)
+    end
+
+    def string?(types)
+      types.include?(String)
+    end
+  end
+end

data/lib/dwca_content_analyzer/csv_converters.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+CSV::Converters[:safe_numeric] = lambda do |field|
+  case field.strip
+  when /^-?[0-9]+$/
+    field.to_i
+  when /^-?[0-9]*\.[0-9]+$/
+    field.to_f
+  else
+    field
+  end
+end
+
+CSV::Converters[:blank_to_nil] = lambda do |field|
+  field&.empty? ? nil : field
+end

data/lib/dwca_content_analyzer/file_contents.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+require 'csv'
+
+require_relative 'column'
+require_relative 'csv_converters'
+
+#
+module DwCAContentAnalyzer
+  #
+  class FileContents
+    attr_reader :columns
+
+    def initialize(file, detectors = %i[type length])
+      @file = file
+      @detectors = detectors
+      @columns = analyze
+    end
+
+    private
+
+    def analyze
+      table = load_table @file
+      table.by_col!.map do |col|
+        header = col[0]
+        contents = col[1]
+        Column.new(header.to_i, contents, *@detectors)
+      end
+    end
+
+    # reads the first line of the CSV file
+    # returns the columns indices as an array
+    def headers(file)
+      Array.new(CSV.open(file, &:readline).size) { |i| i.to_s }
+    end
+
+    def load_table(file)
+      CSV.read(file,
+               headers: headers(file),
+               converters: %i[blank_to_nil safe_numeric date])
+    end
+  end
+end

data/lib/dwca_content_analyzer/file_set.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+require_relative 'file_contents'
+
+#
+module DwCAContentAnalyzer
+  #
+  class FileSet
+    attr_reader :columns
+
+    def initialize(files, detectors = %i[type length])
+      @detectors = detectors
+      @columns = analyze files
+    end
+
+    private
+
+    def analyze(files)
+      consolidate(files.map { |file| columns_for file }.flatten)
+    end
+
+    def consolidate(files)
+      files.group_by(&:index).map do |index, column|
+        length = column.map(&:length).max
+        types = column.map(&:type).uniq
+        { index: index,
+          length: length,
+          type: common_type(types)&.to_s&.underscore }
+      end
+    end
+
+    def columns_for(file)
+      FileContents.new(file, @detectors).columns
+    end
+
+    def common_type(types)
+      if types.size == 1
+        types.first
+      elsif types.size == 2 && types.include?(Float) && types.include?(Integer)
+        Float
+      else
+        String
+      end
+    end
+  end
+end

data/lib/dwcr/dynamic_model_queryable.rb

@@ -0,0 +1,100 @@
+# frozen_string_literal: true
+
+# convenience methods for the dynamic models
+module DynamicModelQueryable
+  # convenience class methods for the dynamic models
+  module DynamicModelClassQueryable
+    # Returns the Metaschema::Attribute for a +column_name+ (the header of the
+    # column, can be passed as Symbol or String).
+    def attribute_for(column_name)
+      entity.attributes_dataset.first(name: column_name.to_s)
+    end
+
+    # Returns the Metschema::Entity the class belongs to.
+    def entity
+      @entity
+    end
+
+    # Returns a nested array of all terms in the order values
+    # will be returned by #to_a. Each item in the nested array will be an array
+    # with the entity term at index 0 and the attribute term at index 1.
+    def template(keys = :term)
+      tmpl = columns.map do |column|
+        next unless attribute = attribute_for(column)
+        [attribute.send(keys), entity.send(keys)]
+      end.compact
+      return tmpl.compact unless entity.is_core
+      entity.extensions.each do |xtn|
+        tmpl << xtn.model_get.template(keys)
+      end
+      tmpl
+    end
+  end
+
+  # Extends the class that DynamicModelQueryable is mixed in with
+  # DynamicModelClassQueryable
+  def self.included(host_class)
+    host_class.extend(DynamicModelClassQueryable)
+  end
+
+  # Returns the core row for +self+. Will return +nil+ if +self+ is the core.
+  def core_row
+    return nil if entity.is_core
+    send(entity.core.name)
+  end
+
+  # Returns an array of all related extension rows for +self+. Will return +nil+
+  # if +self+ is an extension.
+  def extension_rows
+    return nil unless entity.is_core
+    entity.extensions.map { |xtn| send(xtn.table_name) }.flatten
+  end
+
+  # Returns a value hash for +self+ without primary or foreign keys.
+  def row_values
+    keys_to_delete = %i[id entity_id]
+    keys_to_delete.push(entity.core&.foreign_key).compact
+    to_hash.clone.delete_if { |key, _| keys_to_delete.include? key }
+  end
+
+  # Returns a nested array of values only in consistent order.
+  def to_a
+    row_array = row_values.map { |_key, value| value }
+    return row_array unless entity.is_core
+    entity.extensions.inject(row_array) do |memo, xtn|
+      memo << send(xtn.table_name).map(&:to_a)
+    end
+  end
+
+  # Returns a value hash for the row without primary or foreign keys
+  # where the keys in the hash can be the _term_, _baseterm_, or _name_
+  # of the attributes, depending on the argument given
+  def to_hash_with(keys = :term)
+    return row_values if keys == :name
+    row_values.transform_keys do |key|
+      attribute = entity.attributes_dataset.first(name: key.to_s)
+      attribute.send(keys)
+    end
+  end
+
+  # Returns the #full_record for +self+ as JSON.
+  def to_json
+    JSON.generate(to_record)
+  end
+
+  # Returns the full record (current row and all related rows) for +self+
+  # as a hash with +keys+ (+:term+, +:baseterm+, or +:name+).
+  def to_record(keys: :term)
+    record_hash = to_hash_with(keys)
+    if entity.is_core
+      extension_rows.each do |row|
+        key = row.entity.send(keys)
+        record_hash[key] ||= []
+        record_hash[key] << row.to_hash_with(keys)
+      end
+    else
+      record_hash[entity.core.send(keys)] = core_row.to_hash_with(keys)
+    end
+    record_hash
+  end
+end

data/lib/dwcr/dynamic_models.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+#
+module DwCR
+  # Creates a Sequel::Model class for a Entity instance
+  # adds all associations given for the Entity instance
+  def self.create_model(entity)
+    model_class = Class.new(Sequel::Model(entity.table_name)) do
+      include DynamicModelQueryable
+      @entity = entity
+      entity.model_associations.each do |association|
+        associate(*association)
+        next if association[0] == :many_to_one
+        plugin :association_dependencies
+        add_association_dependencies(association[1] => :destroy)
+      end
+
+      define_singleton_method(:finalize) do
+        @entity = nil
+        instance_methods(false).each { |method| remove_method(method) }
+        Module.nesting.last.send(:remove_const, entity.class_name)
+      end
+    end
+    const_set entity.class_name, model_class
+    model_class
+  end
+end

data/lib/dwcr/inflections.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+INFLECTIONS = [
+  [:singular, /(media)$/i, '\1'],
+  [:plural, /(media)$/i, '\1'],
+  [:irregular, 'taxon', 'taxa'],
+  [:uncountable, 'metadata']
+].freeze
+
+Sequel.inflections do |inflect|
+  INFLECTIONS.each { |i| inflect.send(*i) }
+end
+
+String.inflections do |inflect|
+  INFLECTIONS.each { |i| inflect.send(*i) }
+end

data/lib/dwcr/metaschema/archive.rb

@@ -0,0 +1,86 @@
+# frozen_string_literal: true
+
+#
+module DwCR
+  #
+  module Metaschema
+    # This class represents the DarwinCoreArchive's _meta.xml_ file
+    # * +name+: the name for the DarwinCoreArchive
+    #   the default is the directory name given in the path
+    # * +path+: the full path of the directory of the DarwinCoreArchives
+    # * +xmlns+: the XML Namespace
+    #   default: 'http://rs.tdwg.org/dwc/text/'
+    # * +xmlns__xs+: the schema namespace prefix (xmlns:xs),
+    #   default: 'http://www.w3.org/2001/XMLSchema'
+    # * +xmln__xsi+: schema instance namespace prefix (xmln:xsi),
+    #   default: 'http://www.w3.org/2001/XMLSchema-instance'
+    # * +xsi__schema_location+ (xsi:schemaLocation)
+    #   default: 'http://rs.tdwg.org/dwc/text/
+    #             http://rs.tdwg.org/dwc/text/tdwg_dwc_text.xsd'
+    # * *#entities*:
+    #   the associated Entity objects
+    # * *#core*:
+    #   the associated Entity object that is the core node in the DwCA
+    class Archive < Sequel::Model
+      include XMLParsable
+
+      ensure_core = lambda do |entity|
+        self.core_id = entity.id
+        entity.is_core = true
+      end
+
+      ensure_not_core = lambda do |ent, attr|
+        raise 'adding an extension without a core' unless ent.core
+        attr.is_core = false
+        attr.core = ent.core
+      end
+
+      one_to_many :entities
+      many_to_one :core, class: 'Entity',
+                         class_namespace: 'DwCR::Metaschema',
+                         key: :core_id,
+                         setter: ensure_core
+      one_to_many :extensions, class: 'Entity',
+                               class_namespace: 'DwCR::Metaschema',
+                               key: :archive_id,
+                               conditions: { is_core: false },
+                               before_add: ensure_not_core
+
+      # Methods to add records to the :entities association form xml
+
+      # Gets _core_ and _extension_ nodes from the xml
+      # calls #add_entity_from(xml) to create
+      # Entity instances to the Archive for every node
+      # adds the foreign key field (_coreid_) to any _extension_
+      def load_nodes_from(xml)
+        self.core = add_entity_from xml.css('core').first
+        core.save
+        xml.css('extension').each do |node|
+          extn = add_entity_from node
+          extn.add_attribute(name: 'coreid', index: index_from(node))
+          add_extension(extn)
+        end
+        save
+      end
+
+      private
+
+      # Sequel Model hook that creates a default +name+
+      # from the +term+ if present
+      def before_create
+        self.name ||= path&.split('/')&.last
+        super
+      end
+
+      # Creates a Entity instance from xml node (_core_ or _extension_)
+      # adds Attribute instances for any _field_ given
+      # adds ContentFile instances for any child node of _files_
+      def add_entity_from(xml)
+        entity = add_entity(values_from(xml, :term, :key_column))
+        xml.css('field').each { |field| entity.add_attribute_from(field) }
+        entity.add_files_from(xml, path: path)
+        entity
+      end
+    end
+  end
+end