dwc-archive 0.9.5 → 1.1.0

data/lib/dwc_archive/metadata.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+class DarwinCore
+  # Represents data from EML (Ecological Metadata Language) file
+  class Metadata
+    attr_reader :metadata
+    alias data metadata
+
+    def initialize(archive = nil)
+      @archive = archive
+      @metadata = @archive.eml
+    end
+
+    def id
+      fix_nil { @metadata[:eml][:dataset][:attributes][:id] }
+    end
+
+    def package_id
+      fix_nil { @metadata.data[:eml][:attributes][:packageId] }
+    end
+
+    def title
+      fix_nil { @metadata[:eml][:dataset][:title] }
+    end
+
+    def authors
+      return nil unless defined?(@metadata[:eml][:dataset][:creator])
+      authors = [@metadata[:eml][:dataset][:creator]].flatten
+      authors.map do |au|
+        { first_name: au[:individualName][:givenName],
+          last_name: au[:individualName][:surName],
+          email: au[:electronicMailAddress] }
+      end
+    end
+
+    def abstract
+      fix_nil { @metadata[:eml][:dataset][:abstract] }
+    end
+
+    def citation
+      fix_nil { @metadata[:eml][:additionalMetadata][:metadata][:citation] }
+    end
+
+    def url
+      fix_nil { @metadata[:eml][:dataset][:distribution][:online][:url] }
+    end
+
+    private
+
+    def fix_nil
+      yield
+    rescue NoMethodError
+      nil
+    end
+  end
+end

data/lib/dwc_archive/taxon_normalized.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+class DarwinCore
+  # Describes normalized taxon
+  class TaxonNormalized
+    attr_accessor :id, :local_id, :global_id, :source, :parent_id,
+                  :classification_path_id, :classification_path,
+                  :linnean_classification_path, :current_name,
+                  :current_name_canonical, :synonyms, :vernacular_names,
+                  :rank, :status
+
+    def initialize
+      @id = @parent_id = @rank = @status = nil
+      @current_name = @current_name_canonical = @source = @local_id = ""
+      @global_id = ""
+      @classification_path = []
+      @classification_path_id = []
+      @synonyms = []
+      @vernacular_names = []
+      @linnean_classification_path = []
+    end
+  end
+end

data/lib/dwc_archive/version.rb

@@ -0,0 +1,6 @@
+# frozen_string_literal: true
+
+# Version constant of the class
+class DarwinCore
+  VERSION = "1.1.0"
+end

data/lib/dwc_archive/xml_reader.rb

@@ -0,0 +1,89 @@
+# frozen_string_literal: true
+
+class DarwinCore
+  # USAGE: Hash.from_xml:(YOUR_XML_STRING)
+  # modified from
+  # http://stackoverflow.com/questions/1230741/
+  # convert-a-nokogiri-document-to-a-ruby-hash/1231297#1231297
+  module XmlReader
+    def self.from_xml(xml_io)
+      result = Nokogiri::XML(xml_io)
+      { result.root.name.to_sym => self::Node.new(result.root).value }
+    end
+
+    # Node is a helper class to parse xml into hash
+    class Node
+      def initialize(node)
+        @node = node
+        @val = {}
+      end
+
+      def value
+        if @node.element?
+          prepare_node_element
+        else
+          prepare(@node.content.to_s)
+        end
+      end
+
+      private
+
+      def prepare_node_element
+        add_attributes
+        add_children if @node.children.size.positive?
+        @val
+      end
+
+      def prepare(data)
+        data.class == String && data.to_i.to_s == data ? data.to_i : data
+      end
+
+      def add_attributes
+        return if @node.attributes.empty?
+        @val[:attributes] = {}
+        @node.attributes.keys.each do |key|
+          add_attribute(@val[:attributes], @node.attributes[key])
+        end
+      end
+
+      def add_attribute(attributes, attribute)
+        attributes[attribute.name.to_sym] = prepare(attribute.value)
+      end
+
+      def add_children
+        @node.children.each do |child|
+          process_child(child)
+        end
+      end
+
+      def process_child(child)
+        value = DarwinCore::XmlReader::Node.new(child).value
+        if child.name == "text"
+          handle_text(child, value)
+        else
+          add_child_to_value(child, value)
+        end
+      end
+
+      def add_child_to_value(child, value)
+        if @val[child.name.to_sym]
+          handle_child_node(child.name.to_sym, value)
+        else
+          @val[child.name.to_sym] = prepare(value)
+        end
+      end
+
+      def handle_child_node(child, val)
+        if @val[child].is_a?(Object::Array)
+          @val[child] << prepare(val)
+        else
+          @val[child] = [@val[child], prepare(val)]
+        end
+      end
+
+      def handle_text(child, val)
+        @val = prepare(val) unless child.next_sibling || child.previous_sibling
+      end
+    end
+  end
+end

data/spec/files/generator_eml.xml

@@ -0,0 +1,47 @@
+<?xml version="1.0"?>
+<eml:eml xmlns:eml="eml://ecoinformatics.org/eml-2.1.1" xmlns:md="eml://ecoinformatics.org/methods-2.1.1" xmlns:proj="eml://ecoinformatics.org/project-2.1.1" xmlns:d="eml://ecoinformatics.org/dataset-2.1.1" xmlns:res="eml://ecoinformatics.org/resource-2.1.1" xmlns:dc="http://purl.org/dc/terms/" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" packageId="1234/2013-12-30::19:45:33" system="http://globalnames.org" xml:lang="en" xsi:schemaLocation="eml://ecoinformatics.org/eml-2.1.1 http://rs.gbif.org/schema/eml-gbif-profile/1.0.1/eml.xsd">
+  <dataset id="1234">
+    <title>Test Classification</title>
+    <license>http://creativecommons.org/licenses/by-sa/3.0/</license>
+    <creator id="1" scope="document">
+      <individualName>
+        <givenName>John</givenName>
+        <surName>Doe</surName>
+      </individualName>
+      <organizationName>Example</organizationName>
+      <positionName>Assistant Professor</positionName>
+      <onlineUrl>http://example.org</onlineUrl>
+      <electronicMailAddress>jdoe@example.com</electronicMailAddress>
+    </creator>
+    <creator id="2" scope="document">
+      <individualName>
+        <givenName>Jane</givenName>
+        <surName>Doe</surName>
+      </individualName>
+      <electronicMailAddress>jane@example.com</electronicMailAddress>
+    </creator>
+    <metadataProvider>
+      <individualName>
+        <givenName>Jim</givenName>
+        <surName>Doe</surName>
+      </individualName>
+      <onlineUrl>http://aggregator.example.org</onlineUrl>
+      <electronicMailAddress>jimdoe@example.com</electronicMailAddress>
+    </metadataProvider>
+    <pubDate>2013-12-30 14:45:33 -0500</pubDate>
+    <abstract>
+      <para>test classification</para>
+    </abstract>
+    <contact>
+      <references>1</references>
+    </contact>
+    <contact>
+      <references>2</references>
+    </contact>
+  </dataset>
+  <additionalMetadata>
+    <metadata>
+      <citation>Test classification: Doe John, Doe Jane, Taxnonmy, 10, 1, 2010</citation>
+    </metadata>
+  </additionalMetadata>
+</eml:eml>

data/spec/files/generator_meta.xml

@@ -0,0 +1,19 @@
+<?xml version="1.0"?>
+<archive xmlns="http://rs.tdwg.org/dwc/text/" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://rs.tdwg.org/dwc/terms/xsd/archive/ http://darwincore.googlecode.com/svn/trunk/text/tdwg_dwc_text.xsd">
+  <core encoding="UTF-8" fieldsTerminatedBy="," fieldsEnclosedBy="&quot;" linesTerminatedBy="&#10;" rowType="http://rs.tdwg.org/dwc/terms/Taxon" ignoreHeaderLines="1">
+    <files>
+      <location>core.csv</location>
+    </files>
+    <id index="0"/>
+    <field term="http://rs.tdwg.org/dwc/terms/parentNameUsageID" index="1"/>
+    <field term="http://rs.tdwg.org/dwc/terms/scientificName" index="2"/>
+    <field term="http://rs.tdwg.org/dwc/terms/taxonRank" index="3"/>
+  </core>
+  <extension encoding="UTF-8" fieldsTerminatedBy="," fieldsEnclosedBy="&quot;" linesTerminatedBy="&#10;" rowType="http://rs.gbif.org/terms/1.0/VernacularName" ignoreHeaderLines="1">
+    <files>
+      <location>vern.csv</location>
+    </files>
+    <coreid index="0"/>
+    <field term="http://rs.tdwg.org/dwc/terms/vernacularName" index="1"/>
+  </extension>
+</archive>

data/spec/lib/classification_normalizer_spec.rb

@@ -0,0 +1,214 @@
+# frozen_string_literal: true
+
+describe DarwinCore::ClassificationNormalizer do
+  subject(:dwca) { DarwinCore.new(file_path) }
+  subject(:normalizer) { DarwinCore::ClassificationNormalizer.new(dwca) }
+
+  let(:file_dir) { File.expand_path("../files", __dir__) }
+  let(:file_path) { File.join(file_dir, file_name) }
+
+  describe ".new" do
+    let(:file_path) { File.join(file_dir, "data.tar.gz") }
+    it do
+      expect(normalizer.is_a?(DarwinCore::ClassificationNormalizer)).to be true
+    end
+  end
+
+  describe "#normalize" do
+    let(:file_name) { "data.tar.gz" }
+
+    it "returns normalized data" do
+      res = normalizer.normalize
+      expect(res).to be normalizer.normalized_data
+    end
+
+    context "flat list" do
+      let(:file_path) { File.join(file_dir, "flat_list.tar.gz") }
+
+      it "returns flat list" do
+        normalizer.normalize
+        expect(normalizer.normalized_data).to be_kind_of Hash
+        expect(normalizer.normalized_data.size).to be > 0
+      end
+    end
+
+    context "synonyms from core" do
+      let(:file_name) { "synonyms_in_core_accepted_name_field.tar.gz" }
+
+      it "ingests synonyms using accepted_name field" do
+        res = normalizer.normalize
+        syn = res.reject { |_, v| v.synonyms.empty? }.values
+        expect(syn.size).to be > 0
+        expect(syn[0].synonyms[0]).to be_kind_of DarwinCore::SynonymNormalized
+      end
+    end
+
+    context "synonyms from extension" do
+      let(:file_name) { "synonyms_in_extension.tar.gz" }
+      it "ingests synonyms from extension" do
+        res = normalizer.normalize
+        syn = res.reject { |_, v| v.synonyms.empty? }.values
+        expect(syn.size).to be > 0
+        expect(syn[0].synonyms[0]).to be_kind_of DarwinCore::SynonymNormalized
+      end
+    end
+
+    context "synonyms are not extensions" do
+      let(:file_name) { "not_synonym_in_extension.tar.gz" }
+
+      it "does not ingest synonyms" do
+        res = normalizer.normalize
+        syn = res.reject { |_, v| v.synonyms.empty? }.values
+        expect(syn).to be_empty
+      end
+    end
+
+    context "with_extensions flag set on false" do
+      let(:file_name) { "synonyms_in_extension.tar.gz" }
+      it "should not harvest extensions" do
+        res = normalizer.normalize(with_extensions: false)
+        syn = res.reject { |_, v| v.synonyms.empty? }.values
+        expect(syn).to be_empty
+      end
+    end
+
+    context "linnean classification in file (class, order etc fields)" do
+      let(:file_name) { "linnean.tar.gz" }
+
+      it "assembles classification" do
+        res = normalizer.normalize
+        expect(res.first[1]).to be_kind_of DarwinCore::TaxonNormalized
+        expect(res.first[1].linnean_classification_path).
+          to eq [["Animalia", :kingdom],
+                 ["Arthropoda", :phylum],
+                 ["Insecta", :class],
+                 ["Diptera", :order],
+                 ["Cecidomyiidae", :family],
+                 ["Resseliella", :genus]]
+      end
+    end
+
+    context "no linnean fields are given" do
+      it "returns empty linnean classification" do
+        res = normalizer.normalize
+        expect(res.first[1]).to be_kind_of DarwinCore::TaxonNormalized
+        expect(res.first[1].linnean_classification_path).to be_empty
+      end
+    end
+
+    context "in the presence of scientificNameAuthorship field" do
+      let(:file_name) { "sci_name_authorship.tar.gz" }
+      it "returns normalized data" do
+        normalizer.normalize
+        expect(normalizer.darwin_core.file_name).
+          to eq "sci_name_authorship.tar.gz"
+        expect(normalizer.normalized_data).to be_kind_of Hash
+        expect(normalizer.normalized_data.size).to be > 0
+        tn = normalizer.normalized_data["leptogastrinae:tid:2688"]
+        expect(tn.current_name).to eq "Leptogaster fornicata Martin, 1957"
+        expect(tn.current_name_canonical).to eq "Leptogaster fornicata"
+      end
+    end
+
+    context "when scientificNameAuthorship duplicates author info" do
+      let(:file_name) { "sci_name_authorship_dup.tar.gz" }
+      it "returns normalized data" do
+        normalizer.normalize
+        expect(normalizer.darwin_core.file_name).
+          to eq "sci_name_authorship_dup.tar.gz"
+        expect(normalizer.normalized_data).to be_kind_of Hash
+        expect(normalizer.normalized_data.size).to be > 0
+        tn = normalizer.normalized_data["leptogastrinae:tid:2688"]
+        expect(tn.current_name).to eq "Leptogaster fornicata Martin, 1957"
+        expect(tn.current_name_canonical).to eq "Leptogaster fornicata"
+      end
+    end
+
+    context "coreid is empty" do
+      let(:file_name) { "empty_coreid.tar.gz" }
+      it "should ingest information" do
+        res = normalizer.normalize
+        expect(normalizer.darwin_core.file_name).
+          to eq "empty_coreid.tar.gz"
+        tn = res["Taxon9"]
+        expect(tn.current_name).to eq "Amanita phalloides"
+      end
+    end
+
+    context "vernacular locality info" do
+      let(:file_name) { "language_locality.tar.gz" }
+      it "should ingest locality and language" do
+        res = normalizer.normalize
+        tn = res["leptogastrinae:tid:42"]
+        vn = tn.vernacular_names[0]
+        expect(vn.language).to eq "en"
+        expect(vn.locality).to eq "New England"
+      end
+    end
+  end
+
+  describe "#name_strings" do
+    let(:file_path) { File.join(file_dir, "flat_list.tar.gz") }
+
+    context "before running #normalize" do
+      it "is empty" do
+        expect(normalizer.name_strings).to be_empty
+      end
+    end
+
+    context "after running #normalize" do
+      let(:normalized) { normalizer.tap(&:normalize) }
+
+      context "default attibutes" do
+        it "returns array" do
+          expect(normalized.name_strings).to be_kind_of Array
+          expect(normalized.name_strings.size).to be > 1
+        end
+      end
+
+      context "with_hash attribute" do
+        it "returns hash" do
+          strings = normalized.name_strings(with_hash: true)
+          expect(strings).to be_kind_of Hash
+          expect(strings.size).to be > 1
+          expect(strings.values.uniq).to eq [1]
+        end
+      end
+    end
+  end
+
+  describe "#vernacular_name_strings" do
+    let(:file_path) { File.join(file_dir, "flat_list.tar.gz") }
+
+    context "before running #normalize" do
+      subject(:vern) { normalizer.vernacular_name_strings }
+
+      it "is empty" do
+        expect(vern).to be_empty
+      end
+    end
+
+    context "after running #normalize" do
+      let(:normalized) { normalizer.tap(&:normalize) }
+      subject(:vern) { normalized.vernacular_name_strings }
+      subject(:vern_w_hash) do
+        normalized.vernacular_name_strings(with_hash: true)
+      end
+
+      context "default attibutes" do
+        it "returns array" do
+          expect(vern).to be_kind_of Array
+          expect(vern.size).to be > 0
+        end
+      end
+
+      context "with_hash attribute" do
+        it "returns hash" do
+          expect(vern_w_hash).to be_kind_of Hash
+          expect(vern_w_hash.size).to be > 0
+          expect(vern_w_hash.values.uniq).to eq [1]
+        end
+      end
+    end
+  end
+end