dwc-archive 0.9.11 → 1.0.1

data/.rvmrc

@@ -1 +0,0 @@
-rvm use ruby-1.9.3-p392@dwc-archive --create

data/lib/dwc-archive.rb

@@ -1,107 +0,0 @@
-# encoding: UTF-8
-
-recent_ruby = RUBY_VERSION >= '1.9.1'
-raise "IMPORTANT: dwc-archive gem  requires ruby >= 1.9.1" unless recent_ruby
-
-require 'fileutils'
-require 'ostruct'
-require 'digest'
-require 'csv'
-require 'logger'
-require 'nokogiri'
-require_relative 'dwc-archive/xml_reader'
-require_relative 'dwc-archive/ingester'
-require_relative 'dwc-archive/errors'
-require_relative 'dwc-archive/expander'
-require_relative 'dwc-archive/archive'
-require_relative 'dwc-archive/core'
-require_relative 'dwc-archive/extension'
-require_relative 'dwc-archive/metadata'
-require_relative 'dwc-archive/generator'
-require_relative 'dwc-archive/generator_meta_xml'
-require_relative 'dwc-archive/generator_eml_xml'
-require_relative 'dwc-archive/classification_normalizer'
-require_relative 'dwc-archive/version'
-
-class DarwinCore
-
-  DEFAULT_TMP_DIR = "/tmp"
-
-  attr_reader :archive, :core, :metadata, :extensions, 
-    :classification_normalizer
-  alias :eml :metadata
-
-
-  def self.nil_field?(field)
-    return true if [nil, '', '/N'].include?(field)
-    false
-  end
-
-  def self.clean_all(tmp_dir = DEFAULT_TMP_DIR)
-    Dir.entries(tmp_dir).each do |entry|
-      path = File.join(tmp_dir, entry)
-      if FileTest.directory?(path) && entry.match(/^dwc_[\d]+$/)
-        FileUtils.rm_rf(path)
-      end
-    end
-  end
-
-  def self.logger
-    @@logger ||= Logger.new(nil)
-  end
-
-  def self.logger=(logger)
-    @@logger = logger
-  end
-
-  def self.logger_reset
-    self.logger = Logger.new(nil)
-  end
-
-  def self.logger_write(obj_id, message, method = :info)
-    self.logger.send(method, "|%s|%s|" % [obj_id, message])
-  end
-
-  def initialize(dwc_path, tmp_dir = DEFAULT_TMP_DIR)
-    @dwc_path = dwc_path
-    @archive = DarwinCore::Archive.new(@dwc_path, tmp_dir)
-    @core = DarwinCore::Core.new(self)
-    @metadata = DarwinCore::Metadata.new(@archive)
-    @extensions = get_extensions
-  end
-
-  def file_name
-    File.split(@dwc_path).last
-  end
-
-  def path
-    File.expand_path(@dwc_path)
-  end
-
-  # generates a hash from a classification data with path to each node, 
-  # list of synonyms and vernacular names.
-  def normalize_classification
-    return nil unless has_parent_id?
-    @classification_normalizer ||= DarwinCore::ClassificationNormalizer.
-      new(self)
-    @classification_normalizer.normalize
-  end
-
-  def has_parent_id?
-    !!@core.fields.join('|').downcase.match(/highertaxonid|parentnameusageid/)
-  end
-
-  def checksum
-    Digest::SHA1.hexdigest(open(@dwc_path).read)
-  end
-
-  private
-  def get_extensions
-    res = []
-    root_key = @archive.meta.keys[0]
-    ext = @archive.meta[root_key][:extension]
-    return [] unless ext
-    ext = [ext] if ext.class != Array
-    ext.map { |e| DarwinCore::Extension.new(self, e) }
-  end
-end

data/lib/dwc-archive/archive.rb

@@ -1,40 +0,0 @@
-class DarwinCore
-  class Archive
-    attr_reader :meta, :eml
-
-    def initialize(archive_path, tmp_dir)
-      @archive_path = archive_path
-      @tmp_dir = tmp_dir
-      @expander = DarwinCore::Expander.new(@archive_path, @tmp_dir)
-      @expander.unpack
-      if valid?
-        @meta = DarwinCore::XmlReader.
-          from_xml(open(File.join(@expander.path, 'meta.xml')))
-        @eml = files.include?("eml.xml") ? 
-          DarwinCore::XmlReader.
-            from_xml(open(File.join(@expander.path, 'eml.xml'))) : nil
-      else
-        clean
-        raise InvalidArchiveError
-      end
-    end
-
-    def valid?
-      valid = true
-      valid = valid && @expander.path && FileTest.exists?(@expander.path)
-      valid = valid && files && files.include?('meta.xml')
-    end
-    
-    def files
-      @expander.files
-    end
-
-    def files_path
-      @expander.path
-    end
-
-    def clean
-      @expander.clean
-    end
-  end
-end

data/lib/dwc-archive/classification_normalizer.rb

@@ -1,427 +0,0 @@
-# encoding: utf-8
-require 'parsley-store'
-
-class DarwinCore
-
-  class TaxonNormalized
-    attr_accessor :id, :local_id, :global_id, :source, :parent_id,
-      :classification_path_id, :classification_path,
-      :linnean_classification_path, :current_name, :current_name_canonical,
-      :synonyms, :vernacular_names, :rank, :status
-
-    def initialize
-      @id = @parent_id = @rank = @status = nil
-      @current_name = ''
-      @current_name_canonical = ''
-      @source = ''
-      @local_id = ''
-      @global_id = ''
-      @classification_path = []
-      @classification_path_id = []
-      @synonyms = []
-      @vernacular_names = []
-      @linnean_classification_path = []
-    end
-
-  end
-
-  class GnubTaxon < TaxonNormalized
-    attr_accessor :uuid, :uuid_path
-
-    def initialize
-      super
-      @uuid = nil
-      @uuid_path = []
-    end
-  end
-
-  class SynonymNormalized < Struct.new(:id, :name, :canonical_name,
-                                       :status, :source, :local_id,
-                                       :global_id);end
-  class VernacularNormalized < Struct.new(:name, :language, :locality,
-                                          :country_code);end
-
-  class ClassificationNormalizer
-    attr_reader :error_names, :tree, :normalized_data
-
-    def initialize(dwc_instance)
-      @dwc = dwc_instance
-      @core_fields = get_fields(@dwc.core)
-      @extensions = @dwc.extensions.map { |e| [e, get_fields(e)] }
-      @normalized_data = {}
-      @synonyms = {}
-      @parser = ParsleyStore.new(1,2)
-      @name_strings = {}
-      @vernacular_name_strings = {}
-      @error_names = []
-      @tree = {}
-    end
-
-    def darwin_core
-      @dwc
-    end
-
-    def add_name_string(name_string)
-      @name_strings[name_string] = 1 unless @name_strings[name_string]
-    end
-
-    def add_vernacular_name_string(name_string)
-      unless @vernacular_name_strings[name_string]
-        @vernacular_name_strings[name_string] = 1
-      end
-    end
-
-    def name_strings(opts = {})
-      process_strings(@name_strings, opts)
-    end
-
-    def vernacular_name_strings(opts = {})
-      process_strings(@vernacular_name_strings, opts)
-    end
-
-    def normalize(opts = {})
-      opts = { :with_canonical_names => true,
-        :with_extensions => true }.merge(opts)
-      @with_canonical_names = !!opts[:with_canonical_names]
-      DarwinCore.logger_write(@dwc.object_id,
-                              'Started normalization of the classification')
-      ingest_core
-      DarwinCore.logger_write(@dwc.object_id,
-        'Calculating the classification parent/child paths')
-      has_parent_id? ?
-        calculate_classification_path :
-        @normalized_data.keys.each { |id| @tree[id] = {} }
-      DarwinCore.logger_write(@dwc.object_id, 'Ingesting data from extensions')
-      if !!opts[:with_extensions]
-        ingest_extensions
-      end
-      @normalized_data
-    end
-
-  private
-
-    def process_strings(strings, opts)
-      opts = { with_hash: false }.merge(opts)
-      if !!opts[:with_hash]
-        strings
-      else
-        strings.keys
-      end
-    end
-
-    def get_canonical_name(a_scientific_name)
-      if @with_canonical_names
-        canonical_name = @parser.parse(a_scientific_name,
-                                       :canonical_only => true)
-        canonical_name.to_s.empty? ? a_scientific_name : canonical_name
-      else
-        nil
-      end
-    end
-
-    def get_fields(element)
-      data = element.fields.inject({}) do |res, f|
-        field = f[:term].split('/')[-1]
-        field = field ? field.downcase.to_sym : ''
-        res[field] = f[:index].to_i
-        res
-      end
-      data[:id] = element.id[:index] if element.id
-      data
-    end
-
-    def status_synonym?(status)
-      status && !!status.match(/^syn/)
-    end
-
-    def add_synonym_from_core(taxon_id, row)
-      @synonyms[row[@core_fields[:id]]] = taxon_id
-      taxon = @normalized_data[row[taxon_id]] ?
-        @normalized_data[row[taxon_id]] :
-        @normalized_data[row[taxon_id]] = DarwinCore::TaxonNormalized.new
-      synonym = SynonymNormalized.new(
-        row[@core_fields[:id]],
-        row[@core_fields[:scientificname]],
-        row[@core_fields[:canonicalname]],
-        @core_fields[:taxonomicstatus] ?
-          row[@core_fields[:taxonomicstatus]] :
-          nil,
-        @core_fields[:source] ? row[@core_fields[:source]] : nil,
-        @core_fields[:localid] ? row[@core_fields[:localid]] : nil,
-        @core_fields[:globalid] ? row[@core_fields[:globalid]] : nil,
-      )
-      taxon.synonyms <<  synonym
-      add_name_string(synonym.name)
-      add_name_string(synonym.canonical_name)
-    end
-
-    def set_scientific_name(row, fields)
-      row[fields[:scientificname]] = 'N/A' unless row[fields[:scientificname]]
-      canonical_name = nil
-      scientific_name = row[fields[:scientificname]].strip
-      if separate_canonical_and_authorship?(row, fields)
-        if @with_canonical_names
-          canonical_name = row[fields[:scientificname]].strip
-        end
-        scientific_name += " #{row[fields[:scientificnameauthorship]].strip}"
-      else
-        if @with_canonical_names
-          canonical_name = get_canonical_name(row[fields[:scientificname]])
-        end
-      end
-      fields[:canonicalname] = row.size
-      row << canonical_name
-      row[fields[:scientificname]] = scientific_name
-    end
-
-    def separate_canonical_and_authorship?(row, fields)
-      authorship = ''
-      if fields[:scientificnameauthorship]
-        authorship = row[fields[:scientificnameauthorship]].to_s.strip
-      end
-      !(authorship.empty? || row[fields[:scientificname]].index(authorship))
-    end
-
-    def ingest_core
-      @normalized_data = {}
-      has_name_and_id = @core_fields[:id] && @core_fields[:scientificname]
-      raise DarwinCore::CoreFileError.new('Darwin Core core fields must ' +
-        'contain taxon id and scientific name') unless has_name_and_id
-      @dwc.core.read do |rows|
-        rows[1].each do |error|
-          @error_names << { :data => error,
-            :error => :reading_or_encoding_error }
-        end
-        rows[0].each do |r|
-          set_scientific_name(r, @core_fields)
-          #core has AcceptedNameUsageId
-          if @core_fields[:acceptednameusageid] &&
-            r[@core_fields[:acceptednameusageid]] &&
-            r[@core_fields[:acceptednameusageid]] != r[@core_fields[:id]]
-            add_synonym_from_core(@core_fields[:acceptednameusageid], r)
-          elsif !@core_fields[:acceptednameusageid] &&
-            @core_fields[:taxonomicstatus] &&
-            status_synonym?(r[@core_fields[:taxonomicstatus]])
-            add_synonym_from_core(parent_id, r) if has_parent_id?
-          else
-            unless @normalized_data[r[@core_fields[:id]]]
-              if gnub_archive?
-                new_taxon = DarwinCore::GnubTaxon.new
-              else
-                new_taxon = DarwinCore::TaxonNormalized.new
-              end
-              @normalized_data[r[@core_fields[:id]]] = new_taxon
-            end
-            taxon = @normalized_data[r[@core_fields[:id]]]
-            if gnub_archive?
-              taxon.uuid = r[@core_fields[:originalnameusageid]]
-              taxon.uuid_path = r[@core_fields[:originalnameusageidpath]].
-                split('|')
-            end
-            taxon.id = r[@core_fields[:id]]
-            taxon.current_name = r[@core_fields[:scientificname]]
-            taxon.current_name_canonical = r[@core_fields[:canonicalname]]
-            taxon.parent_id = has_parent_id? ? r[parent_id] : nil
-            taxon.rank = r[@core_fields[:taxonrank]] if @core_fields[:taxonrank]
-            if @core_fields[:taxonomicstatus]
-              taxon.status = r[@core_fields[:taxonomicstatus]]
-            end
-            taxon.source = r[@core_fields[:source]] if @core_fields[:source]
-            taxon.local_id = r[@core_fields[:localid]] if @core_fields[:localid]
-            if @core_fields[:globalid]
-              taxon.global_id = r[@core_fields[:globalid]]
-            end
-            taxon.linnean_classification_path =
-              get_linnean_classification_path(r, taxon)
-            add_name_string(taxon.current_name)
-            has_canonical = taxon.current_name_canonical &&
-              !taxon.current_name_canonical.empty?
-            add_name_string(taxon.current_name_canonical) if has_canonical
-          end
-        end
-      end
-    end
-
-    def has_parent_id?
-      @has_parent_id ||= @core_fields.has_key?(:highertaxonid) ||
-        @core_fields.has_key?(:parentnameusageid)
-    end
-
-    def parent_id
-      parent_id_field = @core_fields[:highertaxonid] ||
-        @core_fields[:parentnameusageid]
-    end
-
-    def calculate_classification_path
-      @paths_num = 0
-      @normalized_data.each do |taxon_id, taxon|
-        next if !taxon.classification_path_id.empty?
-        res = get_classification_path(taxon)
-        next if res == 'error'
-      end
-    end
-
-    def get_classification_path(taxon)
-      return if !taxon.classification_path_id.empty?
-      @paths_num += 1
-      if @paths_num % 10000 == 0
-        DarwinCore.logger_write(@dwc.object_id,
-                              "Calculated %s paths" % @paths_num)
-      end
-      current_node = {taxon.id => {}}
-      if DarwinCore.nil_field?(taxon.parent_id)
-        if @with_canonical_names
-          taxon.classification_path << taxon.current_name_canonical
-        end
-        taxon.classification_path_id << taxon.id
-        @tree.merge!(current_node)
-      else
-        parent_cp = parent_cpid = nil
-        if @normalized_data[taxon.parent_id]
-          if @with_canonical_names
-            parent_cp = @normalized_data[taxon.parent_id].classification_path
-          end
-          parent_cpid = @normalized_data[taxon.parent_id].
-            classification_path_id
-        else
-          current_parent = @normalized_data[@synonyms[taxon.parent_id]]
-          if current_parent
-            error = 'WARNING: The parent of the taxon ' +
-              "\'#{taxon.current_name}\' is deprecated"
-            @error_names << {:data => taxon,
-              :error => :deprecated_parent,
-              :current_parent => current_parent }
-
-            if @with_canonical_names
-              parent_cp = current_parent.classification_path
-            end
-            parent_cpid = current_parent.classification_path_id
-          else
-            error = 'WARNING: The parent of the taxon ' +
-              "\'#{taxon.current_name}\' not found"
-            @error_names << {:data => taxon,
-              :error => :deprecated_parent, :current_parent => nil}
-          end
-        end
-        return 'error' unless parent_cpid
-        if parent_cpid.empty?
-          res = 'error'
-          begin
-            res = get_classification_path(@normalized_data[taxon.parent_id])
-          rescue SystemStackError
-            @error_names << {:data => taxon,
-              :error => :too_deep_hierarchy, :current_parent => nil}
-          end
-          return res if res == 'error'
-          if @with_canonical_names
-            taxon.classification_path += @normalized_data[taxon.parent_id].
-              classification_path + [taxon.current_name_canonical]
-          end
-          taxon.classification_path_id += @normalized_data[taxon.parent_id].
-            classification_path_id + [taxon.id]
-          parent_node = @normalized_data[taxon.parent_id].
-            classification_path_id.inject(@tree) {|node, id| node[id]}
-          parent_node.merge!(current_node)
-        else
-          taxon.classification_path += parent_cp +
-            [taxon.current_name_canonical] if @with_canonical_names
-          taxon.classification_path_id += parent_cpid + [taxon.id]
-          parent_node = @normalized_data[taxon.parent_id].
-            classification_path_id.inject(@tree) {|node, id| node[id]}
-          begin
-            parent_node.merge!(current_node)
-          rescue NoMethodError => e
-            DarwinCore.logger_write(@dwc.object_id,
-                              "Error '%s' taxon %s" % [e.message, taxon.id])
-            return 'error'
-          end
-        end
-      end
-    end
-
-    def ingest_extensions
-      @extensions.each do |e|
-        ext, fields = *e
-        ingest_synonyms(e) if (File.split(e[0].file_path).
-                               last.match(/synonym/i) &&
-                               fields.keys.include?(:scientificname))
-        ingest_vernaculars(e) if fields.keys.include? :vernacularname
-      end
-    end
-
-    def ingest_synonyms(extension)
-      DarwinCore.logger_write(@dwc.object_id, 'Ingesting synonyms extension')
-      ext, fields = *extension
-      ext.read do |rows|
-        rows[0].each do |r|
-          set_scientific_name(r, fields)
-          synonym = SynonymNormalized.new(
-            nil,
-            r[fields[:scientificname]],
-            r[fields[:canonicalname]],
-            fields[:taxonomicstatus] ? r[fields[:taxonomicstatus]] : nil,
-            fields[:source] ? r[fields[:source]] : nil,
-            fields[:localid] ? r[fields[:localid]] : nil,
-            fields[:globalid] ? r[fields[:globalid]] : nil,
-          )
-          if @normalized_data[r[fields[:id]]]
-            @normalized_data[r[fields[:id]]].synonyms << synonym
-            add_name_string(synonym.name)
-            add_name_string(synonym.canonical_name)
-          else
-            @error_names << { :taxon => synonym,
-              :error => :synonym_of_unknown_taxa }
-          end
-        end
-      end
-    end
-
-    def ingest_vernaculars(extension)
-      DarwinCore.logger_write(@dwc.object_id,
-                              'Ingesting vernacular names extension')
-      ext, fields = *extension
-      ext.read do |rows|
-        rows[0].each do |r|
-
-          language = nil
-          if fields[:language]
-            language = r[fields[:language]]
-          elsif fields[:languagecode]
-            language = r[fields[:languagecode]]
-          end
-
-          locality = fields[:locality] ? r[fields[:locality]] : nil
-
-          country_code = fields[:countrycode] ? r[fields[:countrycode]] : nil
-
-          vernacular = VernacularNormalized.new(
-            r[fields[:vernacularname]],
-            language,
-            locality,
-            country_code)
-          if @normalized_data[r[fields[:id]]]
-            @normalized_data[r[fields[:id]]].vernacular_names << vernacular
-            add_vernacular_name_string(vernacular.name)
-          else
-            @error_names << { :vernacular_name => vernacular,
-              :error => :vernacular_of_unknown_taxa }
-          end
-        end
-      end
-    end
-
-    #Collect linnean classification path only on species level
-    def get_linnean_classification_path(row, taxon)
-      res = []
-      [:kingdom, :phylum, :class,
-       :order, :family, :genus, :subgenus].each do |clade|
-        res << [row[@core_fields[clade]], clade] if @core_fields[clade]
-      end
-      res
-    end
-
-    def gnub_archive?
-      @core_fields[:originalnameusageidpath]
-    end
-  end
-end