dwc-archive 0.9.10 → 1.1.2

data/lib/dwc_archive/ingester.rb

@@ -0,0 +1,106 @@
+# encoding: utf-8
+class DarwinCore
+  # This module abstracts information for reading csv file to be used
+  # in several classes which need such functionality
+  module Ingester
+    attr_reader :data, :properties, :encoding, :fields_separator, :size
+    attr_reader :file_path, :fields, :line_separator, :quote_character,
+                :ignore_headers
+
+    def size
+      @size ||= init_size
+    end
+
+    def read(batch_size = 10_000)
+      DarwinCore.logger_write(@dwc.object_id, "Reading #{name} data")
+      res = []
+      errors = []
+      args = define_csv_args
+      min_size = @fields.map { |f| f[:index].to_i || 0 }.sort[-1] + 1
+      csv = CSV.new(open(@file_path), args)
+      csv.each_with_index do |r, i|
+        next if @ignore_headers && i == 0
+        min_size > r.size ? errors << r : process_csv_row(res, errors, r)
+        next if i == 0 || i % batch_size != 0
+        DarwinCore.logger_write(@dwc.object_id,
+                                format("Ingested %s records from %s",
+                                       i, name))
+        next unless block_given?
+        yield [res, errors]
+        res = []
+        errors = []
+      end
+      yield [res, errors] if block_given?
+      [res, errors]
+    end
+
+    private
+
+    def define_csv_args
+      args = { col_sep: @field_separator }
+      @quote_character = "\b" if @quote_character.empty?
+      args.merge(quote_char: @quote_character)
+    end
+
+    def name
+      self.class.to_s.split("::")[-1].downcase
+    end
+
+    def process_csv_row(result, errors, row)
+      str = row.join("")
+      str = str.force_encoding("utf-8")
+      if str.encoding.name == "UTF-8" && str.valid_encoding?
+        result << row.map { |f| f.nil? ? nil : f.force_encoding("utf-8") }
+      else
+        errors << row
+      end
+    end
+
+    def init_attributes
+      @properties = @data[:attributes]
+      init_encoding
+      @field_separator = init_field_separator
+      @quote_character = @properties[:fieldsEnclosedBy] || ""
+      @line_separator = @properties[:linesTerminatedBy] || "\n"
+      @ignore_headers = @properties[:ignoreHeaderLines] &&
+                        [1, true].include?(@properties[:ignoreHeaderLines])
+      init_file_path
+      init_fields
+    end
+
+    def init_encoding
+      @encoding = @properties[:encoding] || "UTF-8"
+      accepted_encoding = ["utf-8", "utf8", "utf-16", "utf16"].
+                          include?(@encoding.downcase)
+      fail(
+        DarwinCore::EncodingError,
+        "No support for encodings other than utf-8 or utf-16 at the moment"
+      ) unless accepted_encoding
+    end
+
+    def init_file_path
+      file = @data[:location] ||
+             @data[:attributes][:location] ||
+             @data[:files][:location]
+      @file_path = File.join(@path, file)
+      fail DarwinCore::FileNotFoundError, "No file data" unless @file_path
+    end
+
+    def init_fields
+      @data[:field] = [data[:field]] if data[:field].class != Array
+      @fields = @data[:field].map { |f| f[:attributes] }
+      fail DarwinCore::InvalidArchiveError,
+           "No data fields are found" if @fields.empty?
+    end
+
+    def init_field_separator
+      res = @properties[:fieldsTerminatedBy] || ","
+      res = "\t" if res == "\\t"
+      res
+    end
+
+    def init_size
+      `wc -l #{@file_path}`.match(/^\s*([\d]+)\s/)[1].to_i
+    end
+  end
+end

data/lib/dwc_archive/metadata.rb

@@ -0,0 +1,57 @@
+# frozen_string_literal: true
+
+class DarwinCore
+  # Represents data from EML (Ecological Metadata Language) file
+  class Metadata
+    attr_reader :metadata
+    alias data metadata
+
+    def initialize(archive = nil)
+      @archive = archive
+      @metadata = @archive.eml
+    end
+
+    def id
+      fix_nil { @metadata[:eml][:dataset][:attributes][:id] }
+    end
+
+    def package_id
+      fix_nil { @metadata.data[:eml][:attributes][:packageId] }
+    end
+
+    def title
+      fix_nil { @metadata[:eml][:dataset][:title] }
+    end
+
+    def authors
+      return nil unless defined?(@metadata[:eml][:dataset][:creator])
+
+      authors = [@metadata[:eml][:dataset][:creator]].flatten
+      authors.map do |au|
+        { first_name: au[:individualName][:givenName],
+          last_name: au[:individualName][:surName],
+          email: au[:electronicMailAddress] }
+      end
+    end
+
+    def abstract
+      fix_nil { @metadata[:eml][:dataset][:abstract] }
+    end
+
+    def citation
+      fix_nil { @metadata[:eml][:additionalMetadata][:metadata][:citation] }
+    end
+
+    def url
+      fix_nil { @metadata[:eml][:dataset][:distribution][:online][:url] }
+    end
+
+    private
+
+    def fix_nil
+      yield
+    rescue NoMethodError
+      nil
+    end
+  end
+end

data/lib/dwc_archive/taxon_normalized.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+class DarwinCore
+  # Describes normalized taxon
+  class TaxonNormalized
+    attr_accessor :id, :local_id, :global_id, :source, :parent_id,
+                  :classification_path_id, :classification_path,
+                  :linnean_classification_path, :current_name,
+                  :current_name_canonical, :synonyms, :vernacular_names,
+                  :rank, :status
+
+    def initialize
+      @id = @parent_id = @rank = @status = nil
+      @current_name = @current_name_canonical = @source = @local_id = ""
+      @global_id = ""
+      @classification_path = []
+      @classification_path_id = []
+      @synonyms = []
+      @vernacular_names = []
+      @linnean_classification_path = []
+    end
+  end
+end

data/lib/dwc_archive/version.rb

@@ -0,0 +1,6 @@
+# frozen_string_literal: true
+
+# Version constant of the class
+class DarwinCore
+  VERSION = "1.1.2"
+end

data/lib/dwc_archive/xml_reader.rb

@@ -0,0 +1,90 @@
+# frozen_string_literal: true
+
+class DarwinCore
+  # USAGE: Hash.from_xml:(YOUR_XML_STRING)
+  # modified from
+  # http://stackoverflow.com/questions/1230741/
+  # convert-a-nokogiri-document-to-a-ruby-hash/1231297#1231297
+  module XmlReader
+    def self.from_xml(xml_io)
+      result = Nokogiri::XML(xml_io)
+      { result.root.name.to_sym => self::Node.new(result.root).value }
+    end
+
+    # Node is a helper class to parse xml into hash
+    class Node
+      def initialize(node)
+        @node = node
+        @val = {}
+      end
+
+      def value
+        if @node.element?
+          prepare_node_element
+        else
+          prepare(@node.content.to_s)
+        end
+      end
+
+      private
+
+      def prepare_node_element
+        add_attributes
+        add_children if @node.children.size.positive?
+        @val
+      end
+
+      def prepare(data)
+        data.instance_of?(String) && data.to_i.to_s == data ? data.to_i : data
+      end
+
+      def add_attributes
+        return if @node.attributes.empty?
+
+        @val[:attributes] = {}
+        @node.attributes.each_key do |key|
+          add_attribute(@val[:attributes], @node.attributes[key])
+        end
+      end
+
+      def add_attribute(attributes, attribute)
+        attributes[attribute.name.to_sym] = prepare(attribute.value)
+      end
+
+      def add_children
+        @node.children.each do |child|
+          process_child(child)
+        end
+      end
+
+      def process_child(child)
+        value = DarwinCore::XmlReader::Node.new(child).value
+        if child.name == "text"
+          handle_text(child, value)
+        else
+          add_child_to_value(child, value)
+        end
+      end
+
+      def add_child_to_value(child, value)
+        if @val[child.name.to_sym]
+          handle_child_node(child.name.to_sym, value)
+        else
+          @val[child.name.to_sym] = prepare(value)
+        end
+      end
+
+      def handle_child_node(child, val)
+        if @val[child].is_a?(Object::Array)
+          @val[child] << prepare(val)
+        else
+          @val[child] = [@val[child], prepare(val)]
+        end
+      end
+
+      def handle_text(child, val)
+        @val = prepare(val) unless child.next_sibling || child.previous_sibling
+      end
+    end
+  end
+end

data/spec/files/generator_eml.xml

@@ -0,0 +1,47 @@
+<?xml version="1.0"?>
+<eml:eml xmlns:eml="eml://ecoinformatics.org/eml-2.1.1" xmlns:md="eml://ecoinformatics.org/methods-2.1.1" xmlns:proj="eml://ecoinformatics.org/project-2.1.1" xmlns:d="eml://ecoinformatics.org/dataset-2.1.1" xmlns:res="eml://ecoinformatics.org/resource-2.1.1" xmlns:dc="http://purl.org/dc/terms/" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" packageId="1234/2013-12-30::19:45:33" system="http://globalnames.org" xml:lang="en" xsi:schemaLocation="eml://ecoinformatics.org/eml-2.1.1 http://rs.gbif.org/schema/eml-gbif-profile/1.0.1/eml.xsd">
+  <dataset id="1234">
+    <title>Test Classification</title>
+    <license>http://creativecommons.org/licenses/by-sa/3.0/</license>
+    <creator id="1" scope="document">
+      <individualName>
+        <givenName>John</givenName>
+        <surName>Doe</surName>
+      </individualName>
+      <organizationName>Example</organizationName>
+      <positionName>Assistant Professor</positionName>
+      <onlineUrl>http://example.org</onlineUrl>
+      <electronicMailAddress>jdoe@example.com</electronicMailAddress>
+    </creator>
+    <creator id="2" scope="document">
+      <individualName>
+        <givenName>Jane</givenName>
+        <surName>Doe</surName>
+      </individualName>
+      <electronicMailAddress>jane@example.com</electronicMailAddress>
+    </creator>
+    <metadataProvider>
+      <individualName>
+        <givenName>Jim</givenName>
+        <surName>Doe</surName>
+      </individualName>
+      <onlineUrl>http://aggregator.example.org</onlineUrl>
+      <electronicMailAddress>jimdoe@example.com</electronicMailAddress>
+    </metadataProvider>
+    <pubDate>2013-12-30 14:45:33 -0500</pubDate>
+    <abstract>
+      <para>test classification</para>
+    </abstract>
+    <contact>
+      <references>1</references>
+    </contact>
+    <contact>
+      <references>2</references>
+    </contact>
+  </dataset>
+  <additionalMetadata>
+    <metadata>
+      <citation>Test classification: Doe John, Doe Jane, Taxnonmy, 10, 1, 2010</citation>
+    </metadata>
+  </additionalMetadata>
+</eml:eml>

data/spec/files/generator_meta.xml

@@ -0,0 +1,19 @@
+<?xml version="1.0"?>
+<archive xmlns="http://rs.tdwg.org/dwc/text/" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://rs.tdwg.org/dwc/terms/xsd/archive/ http://darwincore.googlecode.com/svn/trunk/text/tdwg_dwc_text.xsd">
+  <core encoding="UTF-8" fieldsTerminatedBy="," fieldsEnclosedBy="&quot;" linesTerminatedBy="&#10;" rowType="http://rs.tdwg.org/dwc/terms/Taxon" ignoreHeaderLines="1">
+    <files>
+      <location>core.csv</location>
+    </files>
+    <id index="0"/>
+    <field term="http://rs.tdwg.org/dwc/terms/parentNameUsageID" index="1"/>
+    <field term="http://rs.tdwg.org/dwc/terms/scientificName" index="2"/>
+    <field term="http://rs.tdwg.org/dwc/terms/taxonRank" index="3"/>
+  </core>
+  <extension encoding="UTF-8" fieldsTerminatedBy="," fieldsEnclosedBy="&quot;" linesTerminatedBy="&#10;" rowType="http://rs.gbif.org/terms/1.0/VernacularName" ignoreHeaderLines="1">
+    <files>
+      <location>vern.csv</location>
+    </files>
+    <coreid index="0"/>
+    <field term="http://rs.tdwg.org/dwc/terms/vernacularName" index="1"/>
+  </extension>
+</archive>

data/spec/lib/classification_normalizer_spec.rb

@@ -1,223 +1,214 @@
-require_relative '../spec_helper'
-# encoding: utf-8
+# frozen_string_literal: true
 
 describe DarwinCore::ClassificationNormalizer do
-
   subject(:dwca) { DarwinCore.new(file_path) }
   subject(:normalizer) { DarwinCore::ClassificationNormalizer.new(dwca) }
-  
-  let(:file_dir) { File.expand_path('../../files', __FILE__) }
+
+  let(:file_dir) { File.expand_path("../files", __dir__) }
   let(:file_path) { File.join(file_dir, file_name) }
 
-  describe '.new' do
-    let(:file_path) { File.join(file_dir, 'data.tar.gz') }
-    it { expect(normalizer.is_a? DarwinCore::ClassificationNormalizer).
-      to be_true }    
-  end 
+  describe ".new" do
+    let(:file_path) { File.join(file_dir, "data.tar.gz") }
+    it do
+      expect(normalizer.is_a?(DarwinCore::ClassificationNormalizer)).to be true
+    end
+  end
 
-  describe '#normalize' do
-    let(:file_name) { 'data.tar.gz' }
+  describe "#normalize" do
+    let(:file_name) { "data.tar.gz" }
 
-    it 'returns normalized data' do
+    it "returns normalized data" do
       res = normalizer.normalize
       expect(res).to be normalizer.normalized_data
     end
 
+    context "flat list" do
+      let(:file_path) { File.join(file_dir, "flat_list.tar.gz") }
 
-    context 'flat list' do
-      let(:file_path) { File.join(file_dir, 'flat_list.tar.gz') }
-      
-      it 'returns flat list' do
+      it "returns flat list" do
         normalizer.normalize
         expect(normalizer.normalized_data).to be_kind_of Hash
         expect(normalizer.normalized_data.size).to be > 0
       end
     end
 
-    context 'synonyms from core' do
-      let(:file_name) { 'synonyms_in_core_accepted_name_field.tar.gz' }
+    context "synonyms from core" do
+      let(:file_name) { "synonyms_in_core_accepted_name_field.tar.gz" }
 
-      it 'ingests synonyms using accepted_name field' do
+      it "ingests synonyms using accepted_name field" do
         res = normalizer.normalize
-        syn = res.select { |k,v| !v.synonyms.empty? }.
-          map { |k,v| v }
+        syn = res.reject { |_, v| v.synonyms.empty? }.values
         expect(syn.size).to be > 0
         expect(syn[0].synonyms[0]).to be_kind_of DarwinCore::SynonymNormalized
       end
     end
 
-    context 'synonyms from extension' do
-      let(:file_name) { 'synonyms_in_extension.tar.gz' }
-      it 'ingests synonyms from extension' do
+    context "synonyms from extension" do
+      let(:file_name) { "synonyms_in_extension.tar.gz" }
+      it "ingests synonyms from extension" do
         res = normalizer.normalize
-        syn = res.select { |k,v| !v.synonyms.empty? }.
-          map { |k,v| v }
+        syn = res.reject { |_, v| v.synonyms.empty? }.values
         expect(syn.size).to be > 0
         expect(syn[0].synonyms[0]).to be_kind_of DarwinCore::SynonymNormalized
       end
     end
 
-    context 'synonyms are not extensions' do
-      let(:file_name) { 'not_synonym_in_extension.tar.gz' }
+    context "synonyms are not extensions" do
+      let(:file_name) { "not_synonym_in_extension.tar.gz" }
 
-      it 'does not ingest synonyms' do
+      it "does not ingest synonyms" do
         res = normalizer.normalize
-        syn = res.select { |k,v| !v.synonyms.empty? }.
-          map { |k,v| v }
+        syn = res.reject { |_, v| v.synonyms.empty? }.values
         expect(syn).to be_empty
       end
     end
 
-    context 'with_extensions flag set on false' do
-      let(:file_name) { 'synonyms_in_extension.tar.gz' }
-      it 'should not harvest extensions' do
+    context "with_extensions flag set on false" do
+      let(:file_name) { "synonyms_in_extension.tar.gz" }
+      it "should not harvest extensions" do
         res = normalizer.normalize(with_extensions: false)
-        syn = res.select { |k,v| !v.synonyms.empty? }.
-          map { |k,v| v }
+        syn = res.reject { |_, v| v.synonyms.empty? }.values
         expect(syn).to be_empty
       end
     end
 
-    context 'linnean classification in file (class, order etc fields)' do
-      let(:file_name) { 'linnean.tar.gz' }
+    context "linnean classification in file (class, order etc fields)" do
+      let(:file_name) { "linnean.tar.gz" }
 
-      it 'assembles classification' do
+      it "assembles classification" do
         res = normalizer.normalize
         expect(res.first[1]).to be_kind_of DarwinCore::TaxonNormalized
         expect(res.first[1].linnean_classification_path).
-          to eq [["Animalia", :kingdom], 
-                 ["Arthropoda", :phylum], 
-                 ["Insecta", :class], 
-                 ["Diptera", :order], 
-                 ["Cecidomyiidae", :family], 
+          to eq [["Animalia", :kingdom],
+                 ["Arthropoda", :phylum],
+                 ["Insecta", :class],
+                 ["Diptera", :order],
+                 ["Cecidomyiidae", :family],
                  ["Resseliella", :genus]]
-
       end
     end
 
-    context 'no linnean fields are given' do
-      it 'returns empty linnean classification' do
+    context "no linnean fields are given" do
+      it "returns empty linnean classification" do
         res = normalizer.normalize
         expect(res.first[1]).to be_kind_of DarwinCore::TaxonNormalized
         expect(res.first[1].linnean_classification_path).to be_empty
       end
     end
 
-    context 'in the presence of scientificNameAuthorship field' do
-      let(:file_name) { 'sci_name_authorship.tar.gz' }
-      it 'returns normalized data' do
+    context "in the presence of scientificNameAuthorship field" do
+      let(:file_name) { "sci_name_authorship.tar.gz" }
+      it "returns normalized data" do
         normalizer.normalize
         expect(normalizer.darwin_core.file_name).
-          to eq 'sci_name_authorship.tar.gz'
+          to eq "sci_name_authorship.tar.gz"
         expect(normalizer.normalized_data).to be_kind_of Hash
         expect(normalizer.normalized_data.size).to be > 0
-        tn = normalizer.normalized_data['leptogastrinae:tid:2688']
-        expect(tn.current_name).to eq 'Leptogaster fornicata Martin, 1957'
-        expect(tn.current_name_canonical).to eq 'Leptogaster fornicata'
+        tn = normalizer.normalized_data["leptogastrinae:tid:2688"]
+        expect(tn.current_name).to eq "Leptogaster fornicata Martin, 1957"
+        expect(tn.current_name_canonical).to eq "Leptogaster fornicata"
       end
     end
 
-    context 'when scientificNameAuthorship duplicates author info' do
-      let(:file_name) { 'sci_name_authorship_dup.tar.gz' }
-      it 'returns normalized data' do
+    context "when scientificNameAuthorship duplicates author info" do
+      let(:file_name) { "sci_name_authorship_dup.tar.gz" }
+      it "returns normalized data" do
         normalizer.normalize
         expect(normalizer.darwin_core.file_name).
-          to eq 'sci_name_authorship_dup.tar.gz'
+          to eq "sci_name_authorship_dup.tar.gz"
         expect(normalizer.normalized_data).to be_kind_of Hash
         expect(normalizer.normalized_data.size).to be > 0
-        tn = normalizer.normalized_data['leptogastrinae:tid:2688']
-        expect(tn.current_name).to eq 'Leptogaster fornicata Martin, 1957'
-        expect(tn.current_name_canonical).to eq 'Leptogaster fornicata'
+        tn = normalizer.normalized_data["leptogastrinae:tid:2688"]
+        expect(tn.current_name).to eq "Leptogaster fornicata Martin, 1957"
+        expect(tn.current_name_canonical).to eq "Leptogaster fornicata"
       end
     end
 
-    context 'coreid is empty' do
-      let(:file_name) { 'empty_coreid.tar.gz' }
-      it 'should ingest information' do
+    context "coreid is empty" do
+      let(:file_name) { "empty_coreid.tar.gz" }
+      it "should ingest information" do
         res = normalizer.normalize
         expect(normalizer.darwin_core.file_name).
-          to eq 'empty_coreid.tar.gz'
-        tn = res['Taxon9']
-        expect(tn.current_name).to eq 'Amanita phalloides'
+          to eq "empty_coreid.tar.gz"
+        tn = res["Taxon9"]
+        expect(tn.current_name).to eq "Amanita phalloides"
       end
     end
 
-    context 'vernacular locality info' do
-      let(:file_name) { 'language_locality.tar.gz' }
-      it 'should ingest locality and language' do
+    context "vernacular locality info" do
+      let(:file_name) { "language_locality.tar.gz" }
+      it "should ingest locality and language" do
         res = normalizer.normalize
-        tn = res['leptogastrinae:tid:42']
+        tn = res["leptogastrinae:tid:42"]
         vn = tn.vernacular_names[0]
-        expect(vn.language).to eq 'en'
-        expect(vn.locality).to eq 'New England'
+        expect(vn.language).to eq "en"
+        expect(vn.locality).to eq "New England"
       end
     end
   end
 
-  describe '#name_strings' do 
-    let(:file_path) { File.join(file_dir, 'flat_list.tar.gz') }
+  describe "#name_strings" do
+    let(:file_path) { File.join(file_dir, "flat_list.tar.gz") }
 
-    context 'before running #normalize' do
-      it 'is empty' do
+    context "before running #normalize" do
+      it "is empty" do
         expect(normalizer.name_strings).to be_empty
       end
     end
-    
-    context 'after running #normalize' do
-      let(:normalized) { normalizer.tap { |n| n.normalize } }
 
-      context 'default attibutes' do
-        it 'returns array' do
+    context "after running #normalize" do
+      let(:normalized) { normalizer.tap(&:normalize) }
+
+      context "default attibutes" do
+        it "returns array" do
           expect(normalized.name_strings).to be_kind_of Array
           expect(normalized.name_strings.size).to be > 1
         end
       end
 
-      context 'with_hash attribute' do
-        it 'returns hash' do
-          strings = normalized.name_strings(with_hash:true)
+      context "with_hash attribute" do
+        it "returns hash" do
+          strings = normalized.name_strings(with_hash: true)
           expect(strings).to be_kind_of Hash
           expect(strings.size).to be > 1
           expect(strings.values.uniq).to eq [1]
         end
       end
     end
-
   end
 
-  describe '#vernacular_name_strings' do
-    let(:file_path) { File.join(file_dir, 'flat_list.tar.gz') }
+  describe "#vernacular_name_strings" do
+    let(:file_path) { File.join(file_dir, "flat_list.tar.gz") }
 
-    context 'before running #normalize' do
+    context "before running #normalize" do
       subject(:vern) { normalizer.vernacular_name_strings }
-      it 'is empty' do
+
+      it "is empty" do
         expect(vern).to be_empty
       end
     end
-    
-    context 'after running #normalize' do
-      let(:normalized) { normalizer.tap { |n| n.normalize } }
+
+    context "after running #normalize" do
+      let(:normalized) { normalizer.tap(&:normalize) }
       subject(:vern) { normalized.vernacular_name_strings }
-      subject(:vern_w_hash) { normalized.
-        vernacular_name_strings(with_hash: true) }
+      subject(:vern_w_hash) do
+        normalized.vernacular_name_strings(with_hash: true)
+      end
 
-      context 'default attibutes' do
-        it 'returns array' do
+      context "default attibutes" do
+        it "returns array" do
           expect(vern).to be_kind_of Array
-          expect(vern.size).to be > 0 
+          expect(vern.size).to be > 0
         end
       end
 
-      context 'with_hash attribute' do
-        it 'returns hash' do
+      context "with_hash attribute" do
+        it "returns hash" do
           expect(vern_w_hash).to be_kind_of Hash
-          expect(vern_w_hash.size).to be > 0 
+          expect(vern_w_hash.size).to be > 0
           expect(vern_w_hash.values.uniq).to eq [1]
         end
       end
-
     end
-
   end
-
 end