dwc-archive 0.9.5 → 0.9.6

data/CHANGELOG

@@ -1,3 +1,7 @@
+0.9.6 Added support for GNUB DwCA files
+
+0.9.4 Gem dependencies updated, added travis support
+
 0.9.0 Migrated code to ruby 1.9.3
 
 0.8.3 Updated outdated exception rasing

data/LICENSE

@@ -1,4 +1,4 @@
-Copyright (c) 2010-2012 Marine Biological Laboratoryj
+Copyright (c) 2010-2012 Marine Biological Laboratory
 
 Permission is hereby granted, free of charge, to any person obtaining
 a copy of this software and associated documentation files (the

data/README.md

@@ -143,6 +143,15 @@ Creating a DarwinCore Archive file
     gen.add_eml_xml(eml)
     gen.pack
 
+Logging
+-------
+
+Gem has ability to show logs of it's events:
+
+    require 'dwc-archive'
+    DarwinCore.logger = Logger.new($stdout)
+
+
 Note on Patches/Pull Requests
 -----------------------------
 

data/Rakefile

@@ -6,7 +6,10 @@ begin
   Jeweler::Tasks.new do |gem|
     gem.name = "dwc-archive"
     gem.summary = %Q{Handler of Darwin Core Archive files}
-    gem.description = %q{Darwin Core Archive is the current standard exchange format for GLobal Names Architecture modules. This gem makes it easy to incorporate files in Darwin Core Archive format into a ruby project.}
+    gem.description = 'Darwin Core Archive is the current standard exchange ' +
+      'format for GLobal Names Architecture modules. ' +
+      'This gem makes it easy to incorporate files in ' +
+      'Darwin Core Archive format into a ruby project.'
     gem.email = "dmozzherin at gmail dot com"
     gem.homepage = "http://github.com/GlobalNamesArchitecture/dwc-archive"
     gem.authors = ["Dmitry Mozzherin"]
@@ -14,11 +17,11 @@ begin
     gem.add_dependency 'parsley-store', ">= 0.3.0"
     gem.add_development_dependency "rspec", ">= 1.2.9"
     gem.add_development_dependency "cucumber", ">= 0"
-    # gem is a Gem::Specification... see http://www.rubygems.org/read/chapter/20 for additional settings
   end
   Jeweler::GemcutterTasks.new
 rescue LoadError
-  puts "Jeweler (or a dependency) not available. Install it with: gem install jeweler"
+  puts 'Jeweler (or a dependency) not available. ' +
+    'Install it with: gem install jeweler'
 end
 
 require 'rspec/core/rake_task'
@@ -40,18 +43,20 @@ begin
   task :features => :check_dependencies
 rescue LoadError
   task :features do
-    abort "Cucumber is not available. In order to run features, you must: sudo gem install cucumber"
+    abort 'Cucumber is not available. In order to run features, ' +
+      'you must: sudo gem install cucumber'
   end
 end
 
-task :default => :spec
+desc 'Run an IRB session with CSL loaded'
+task :irb, [:script] do |t, args|
+  ARGV.clear
+
+  require 'irb'
+  require_relative 'lib/dwc-archive'
 
-# require 'rdoc/task'
-# Rake::RDocTask.new do |rdoc|
-#   version = File.exist?('VERSION') ? File.read('VERSION') : ""
-# 
-#   rdoc.rdoc_dir = 'rdoc'
-#   rdoc.title = "dwc-archive #{version}"
-#   rdoc.rdoc_files.include('README*')
-#   rdoc.rdoc_files.include('lib/**/*.rb')
-# end
+  IRB.conf[:SCRIPT] = args.script
+  IRB.start
+end
+
+task :default => :spec

data/VERSION

@@ -1 +1 @@
-0.9.5
+0.9.6

data/lib/dwc-archive/classification_normalizer.rb

@@ -4,7 +4,10 @@ require 'parsley-store'
 class DarwinCore
 
   class TaxonNormalized
-    attr_accessor :id, :local_id, :global_id, :source, :parent_id, :classification_path_id, :classification_path, :linnean_classification_path, :current_name, :current_name_canonical, :synonyms, :vernacular_names, :rank, :status
+    attr_accessor :id, :local_id, :global_id, :source, :parent_id,
+      :classification_path_id, :classification_path,
+      :linnean_classification_path, :current_name, :current_name_canonical,
+      :synonyms, :vernacular_names, :rank, :status
 
     def initialize
       @id = @parent_id = @rank = @status = nil
@@ -22,8 +25,21 @@ class DarwinCore
 
   end
 
-  class SynonymNormalized < Struct.new(:id, :name, :canonical_name, :status, :source, :local_id, :global_id);end
-  class VernacularNormalized < Struct.new(:name, :language, :locality, :country_code);end
+  class GnubTaxon < TaxonNormalized
+    attr_accessor :uuid, :uuid_path
+
+    def initialize
+      super
+      @uuid = nil
+      @uuid_path = []
+    end
+  end
+
+  class SynonymNormalized < Struct.new(:id, :name, :canonical_name,
+                                       :status, :source, :local_id,
+                                       :global_id);end
+  class VernacularNormalized < Struct.new(:name, :language, :locality,
+                                          :country_code);end
 
   class ClassificationNormalizer
     attr_reader :error_names, :tree, :normalized_data
@@ -46,7 +62,9 @@ class DarwinCore
     end
 
     def add_vernacular_name_string(name_string)
-      @vernacular_name_strings[name_string] = 1 unless @vernacular_name_strings[name_string]
+      unless @vernacular_name_strings[name_string]
+        @vernacular_name_strings[name_string] = 1
+      end
     end
 
     def name_strings(opts = {})
@@ -68,13 +86,18 @@ class DarwinCore
     end
 
     def normalize(opts = {})
-      opts = { :with_canonical_names => true, :with_extensions => true }.merge(opts)
+      opts = { :with_canonical_names => true,
+        :with_extensions => true }.merge(opts)
       @with_canonical_names = !!opts[:with_canonical_names]
-      DarwinCore.logger_write(@dwc.object_id, "Started normalization of the classification")
+      DarwinCore.logger_write(@dwc.object_id,
+                              'Started normalization of the classification')
       ingest_core
-      DarwinCore.logger_write(@dwc.object_id, "Calculating the classification parent/child paths")
-      has_parent_id? ? calculate_classification_path : @normalized_data.keys.each { |id| @tree[id] = {} }
-      DarwinCore.logger_write(@dwc.object_id, "Ingesting data from extensions")
+      DarwinCore.logger_write(@dwc.object_id,
+        'Calculating the classification parent/child paths')
+      has_parent_id? ?
+        calculate_classification_path :
+        @normalized_data.keys.each { |id| @tree[id] = {} }
+      DarwinCore.logger_write(@dwc.object_id, 'Ingesting data from extensions')
       if !!opts[:with_extensions]
         ingest_extensions
       end
@@ -85,7 +108,8 @@ class DarwinCore
 
     def get_canonical_name(a_scientific_name)
       if @with_canonical_names
-        canonical_name = @parser.parse(a_scientific_name, :canonical_only => true)
+        canonical_name = @parser.parse(a_scientific_name,
+                                       :canonical_only => true)
         canonical_name.to_s.empty? ? a_scientific_name : canonical_name
       else
         nil
@@ -93,9 +117,9 @@ class DarwinCore
     end
 
     def get_fields(element)
-      data = element.fields.inject({}) do |res, f| 
+      data = element.fields.inject({}) do |res, f|
         field = f[:term].split('/')[-1]
-        field = field ? field.downcase.to_sym : '' 
+        field = field ? field.downcase.to_sym : ''
         res[field] = f[:index].to_i
         res
       end
@@ -109,12 +133,16 @@ class DarwinCore
 
     def add_synonym_from_core(taxon_id, row)
       @synonyms[row[@core_fields[:id]]] = taxon_id
-      taxon = @normalized_data[row[taxon_id]] ? @normalized_data[row[taxon_id]] : @normalized_data[row[taxon_id]] = DarwinCore::TaxonNormalized.new
+      taxon = @normalized_data[row[taxon_id]] ?
+        @normalized_data[row[taxon_id]] :
+        @normalized_data[row[taxon_id]] = DarwinCore::TaxonNormalized.new
       synonym = SynonymNormalized.new(
         row[@core_fields[:id]],
         row[@core_fields[:scientificname]],
         row[@core_fields[:canonicalname]],
-        @core_fields[:taxonomicstatus] ? row[@core_fields[:taxonomicstatus]] : nil,
+        @core_fields[:taxonomicstatus] ?
+          row[@core_fields[:taxonomicstatus]] :
+          nil,
         @core_fields[:source] ? row[@core_fields[:source]] : nil,
         @core_fields[:localid] ? row[@core_fields[:localid]] : nil,
         @core_fields[:globalid] ? row[@core_fields[:globalid]] : nil,
@@ -129,10 +157,14 @@ class DarwinCore
       canonical_name = nil
       scientific_name = row[fields[:scientificname]].strip
       if separate_canonical_and_authorship?(row, fields)
-        canonical_name = row[fields[:scientificname]].strip if @with_canonical_names
+        if @with_canonical_names
+          canonical_name = row[fields[:scientificname]].strip
+        end
         scientific_name += " #{row[fields[:scientificnameauthorship]].strip}"
       else
-        canonical_name = get_canonical_name(row[fields[:scientificname]]) if @with_canonical_names
+        if @with_canonical_names
+          canonical_name = get_canonical_name(row[fields[:scientificname]])
+        end
       end
       fields[:canonicalname] = row.size
       row << canonical_name
@@ -149,43 +181,72 @@ class DarwinCore
 
     def ingest_core
       @normalized_data = {}
-      raise DarwinCore::CoreFileError.new("Darwin Core core fields must contain taxon id and scientific name") unless (@core_fields[:id] && @core_fields[:scientificname])
+      has_name_and_id = @core_fields[:id] && @core_fields[:scientificname]
+      raise DarwinCore::CoreFileError.new('Darwin Core core fields must ' +
+        'contain taxon id and scientific name') unless has_name_and_id
       @dwc.core.read do |rows|
         rows[1].each do |error|
-          @error_names << { :data => error, :error => :reading_or_encoding_error }
+          @error_names << { :data => error,
+            :error => :reading_or_encoding_error }
         end
         rows[0].each do |r|
           set_scientific_name(r, @core_fields)
           #core has AcceptedNameUsageId
-          if @core_fields[:acceptednameusageid] && r[@core_fields[:acceptednameusageid]] && r[@core_fields[:acceptednameusageid]] != r[@core_fields[:id]]
+          if @core_fields[:acceptednameusageid] &&
+            r[@core_fields[:acceptednameusageid]] &&
+            r[@core_fields[:acceptednameusageid]] != r[@core_fields[:id]]
             add_synonym_from_core(@core_fields[:acceptednameusageid], r)
-          elsif !@core_fields[:acceptednameusageid] && @core_fields[:taxonomicstatus] && status_synonym?(r[@core_fields[:taxonomicstatus]])
+          elsif !@core_fields[:acceptednameusageid] &&
+            @core_fields[:taxonomicstatus] &&
+            status_synonym?(r[@core_fields[:taxonomicstatus]])
             add_synonym_from_core(parent_id, r) if has_parent_id?
           else
-            taxon = @normalized_data[r[@core_fields[:id]]] ? @normalized_data[r[@core_fields[:id]]] : @normalized_data[r[@core_fields[:id]]] = DarwinCore::TaxonNormalized.new
+            unless @normalized_data[r[@core_fields[:id]]]
+              if gnub_archive?
+                new_taxon = DarwinCore::GnubTaxon.new
+              else
+                new_taxon = DarwinCore::TaxonNormalized.new
+              end
+              @normalized_data[r[@core_fields[:id]]] = new_taxon
+            end
+            taxon = @normalized_data[r[@core_fields[:id]]]
+            if gnub_archive?
+              taxon.uuid = r[@core_fields[:originalnameusageid]]
+              taxon.uuid_path = r[@core_fields[:originalnameusageidpath]].
+                split('|')
+            end
             taxon.id = r[@core_fields[:id]]
             taxon.current_name = r[@core_fields[:scientificname]]
             taxon.current_name_canonical = r[@core_fields[:canonicalname]]
             taxon.parent_id = has_parent_id? ? r[parent_id] : nil
             taxon.rank = r[@core_fields[:taxonrank]] if @core_fields[:taxonrank]
-            taxon.status = r[@core_fields[:taxonomicstatus]] if @core_fields[:taxonomicstatus]
+            if @core_fields[:taxonomicstatus]
+              taxon.status = r[@core_fields[:taxonomicstatus]]
+            end
             taxon.source = r[@core_fields[:source]] if @core_fields[:source]
             taxon.local_id = r[@core_fields[:localid]] if @core_fields[:localid]
-            taxon.global_id = r[@core_fields[:globalid]] if @core_fields[:globalid]
-            taxon.linnean_classification_path = get_linnean_classification_path(r, taxon)
+            if @core_fields[:globalid]
+              taxon.global_id = r[@core_fields[:globalid]]
+            end
+            taxon.linnean_classification_path =
+              get_linnean_classification_path(r, taxon)
             add_name_string(taxon.current_name)
-            add_name_string(taxon.current_name_canonical) if taxon.current_name_canonical && !taxon.current_name_canonical.empty?
+            has_canonical = taxon.current_name_canonical &&
+              !taxon.current_name_canonical.empty?
+            add_name_string(taxon.current_name_canonical) if has_canonical
           end
         end
       end
     end
 
     def has_parent_id?
-      @has_parent_id ||= @core_fields.has_key?(:highertaxonid) || @core_fields.has_key?(:parentnameusageid)
+      @has_parent_id ||= @core_fields.has_key?(:highertaxonid) ||
+        @core_fields.has_key?(:parentnameusageid)
     end
 
     def parent_id
-      parent_id_field = @core_fields[:highertaxonid] || @core_fields[:parentnameusageid]
+      parent_id_field = @core_fields[:highertaxonid] ||
+        @core_fields[:parentnameusageid]
     end
 
     def calculate_classification_path
@@ -200,28 +261,43 @@ class DarwinCore
     def get_classification_path(taxon)
       return if !taxon.classification_path_id.empty?
       @paths_num += 1
-      DarwinCore.logger_write(@dwc.object_id, "Calculated %s paths" % @paths_num) if @paths_num % 10000 == 0
+      if @paths_num % 10000 == 0
+        DarwinCore.logger_write(@dwc.object_id,
+                              "Calculated %s paths" % @paths_num)
+      end
       current_node = {taxon.id => {}}
       if DarwinCore.nil_field?(taxon.parent_id)
-        taxon.classification_path << taxon.current_name_canonical if @with_canonical_names
+        if @with_canonical_names
+          taxon.classification_path << taxon.current_name_canonical
+        end
         taxon.classification_path_id << taxon.id
         @tree.merge!(current_node)
       else
         parent_cp = parent_cpid = nil
         if @normalized_data[taxon.parent_id]
-          parent_cp = @normalized_data[taxon.parent_id].classification_path if @with_canonical_names
-          parent_cpid = @normalized_data[taxon.parent_id].classification_path_id
+          if @with_canonical_names
+            parent_cp = @normalized_data[taxon.parent_id].classification_path
+          end
+          parent_cpid = @normalized_data[taxon.parent_id].
+            classification_path_id
         else
           current_parent = @normalized_data[@synonyms[taxon.parent_id]]
           if current_parent
-            error = "WARNING: The parent of the taxon \'#{taxon.current_name}\' is deprecated"
-            @error_names << {:data => taxon, :error => :deprecated_parent, :current_parent => current_parent }
-
-            parent_cp = current_parent.classification_path if @with_canonical_names
+            error = 'WARNING: The parent of the taxon ' +
+              "\'#{taxon.current_name}\' is deprecated"
+            @error_names << {:data => taxon,
+              :error => :deprecated_parent,
+              :current_parent => current_parent }
+
+            if @with_canonical_names
+              parent_cp = current_parent.classification_path
+            end
             parent_cpid = current_parent.classification_path_id
           else
-            error = "WARNING: The parent of the taxon \'#{taxon.current_name}\' not found"
-            @error_names << {:data => taxon, :error => :deprecated_parent, :current_parent => nil}
+            error = 'WARNING: The parent of the taxon ' +
+              "\'#{taxon.current_name}\' not found"
+            @error_names << {:data => taxon,
+              :error => :deprecated_parent, :current_parent => nil}
           end
         end
         return 'error' unless parent_cpid
@@ -230,23 +306,30 @@ class DarwinCore
           begin
             res = get_classification_path(@normalized_data[taxon.parent_id])
           rescue SystemStackError
-            @error_names << {:data => taxon, :error => :too_deep_hierarchy, :current_parent => nil}
+            @error_names << {:data => taxon,
+              :error => :too_deep_hierarchy, :current_parent => nil}
           end
           return res if res == 'error'
           if @with_canonical_names
-            taxon.classification_path += @normalized_data[taxon.parent_id].classification_path + [taxon.current_name_canonical]
+            taxon.classification_path += @normalized_data[taxon.parent_id].
+              classification_path + [taxon.current_name_canonical]
           end
-          taxon.classification_path_id += @normalized_data[taxon.parent_id].classification_path_id + [taxon.id]
-          parent_node = @normalized_data[taxon.parent_id].classification_path_id.inject(@tree) {|node, id| node[id]}
+          taxon.classification_path_id += @normalized_data[taxon.parent_id].
+            classification_path_id + [taxon.id]
+          parent_node = @normalized_data[taxon.parent_id].
+            classification_path_id.inject(@tree) {|node, id| node[id]}
           parent_node.merge!(current_node)
         else
-          taxon.classification_path += parent_cp + [taxon.current_name_canonical] if @with_canonical_names
+          taxon.classification_path += parent_cp +
+            [taxon.current_name_canonical] if @with_canonical_names
           taxon.classification_path_id += parent_cpid + [taxon.id]
-          parent_node = @normalized_data[taxon.parent_id].classification_path_id.inject(@tree) {|node, id| node[id]}
+          parent_node = @normalized_data[taxon.parent_id].
+            classification_path_id.inject(@tree) {|node, id| node[id]}
           begin
             parent_node.merge!(current_node)
           rescue NoMethodError => e
-            DarwinCore.logger_write(@dwc.object_id, "Error '%s' taxon %s" % [e.message, taxon.id])
+            DarwinCore.logger_write(@dwc.object_id,
+                              "Error '%s' taxon %s" % [e.message, taxon.id])
             return 'error'
           end
         end
@@ -256,13 +339,15 @@ class DarwinCore
     def ingest_extensions
       @extensions.each do |e|
         ext, fields = *e
-        ingest_synonyms(e) if (File.split(e[0].file_path).last.match(/synonym/i) && fields.keys.include?(:scientificname))
+        ingest_synonyms(e) if (File.split(e[0].file_path).
+                               last.match(/synonym/i) &&
+                               fields.keys.include?(:scientificname))
         ingest_vernaculars(e) if fields.keys.include? :vernacularname
       end
     end
 
     def ingest_synonyms(extension)
-      DarwinCore.logger_write(@dwc.object_id, "Ingesting synonyms extension")
+      DarwinCore.logger_write(@dwc.object_id, 'Ingesting synonyms extension')
       ext, fields = *extension
       ext.read do |rows|
         rows[0].each do |r|
@@ -281,14 +366,16 @@ class DarwinCore
             add_name_string(synonym.name)
             add_name_string(synonym.canonical_name)
           else
-            @error_names << { :taxon => synonym, :error => :synonym_of_unknown_taxa }
+            @error_names << { :taxon => synonym,
+              :error => :synonym_of_unknown_taxa }
           end
         end
       end
     end
 
     def ingest_vernaculars(extension)
-      DarwinCore.logger_write(@dwc.object_id, "Ingesting vernacular names extension")
+      DarwinCore.logger_write(@dwc.object_id,
+                              'Ingesting vernacular names extension')
       ext, fields = *extension
       ext.read do |rows|
         rows[0].each do |r|
@@ -313,20 +400,25 @@ class DarwinCore
             @normalized_data[r[fields[:id]]].vernacular_names << vernacular
             add_vernacular_name_string(vernacular.name)
           else
-            @error_names << { :vernacular_name => vernacular, :error => :vernacular_of_unknown_taxa }
+            @error_names << { :vernacular_name => vernacular,
+              :error => :vernacular_of_unknown_taxa }
           end
         end
       end
     end
-    
+
     #Collect linnean classification path only on species level
     def get_linnean_classification_path(row, taxon)
       res = []
-      [:kingdom, :phylum, :class, :order, :family, :genus, :subgenus].each do |clade|
-        res << [row[@core_fields[clade]], clade] if @core_fields[clade] 
+      [:kingdom, :phylum, :class,
+       :order, :family, :genus, :subgenus].each do |clade|
+        res << [row[@core_fields[clade]], clade] if @core_fields[clade]
       end
       res
     end
 
+    def gnub_archive?
+      @core_fields[:originalnameusageidpath]
+    end
   end
 end

data/spec/lib/dwc-archive_spec.rb

@@ -49,7 +49,7 @@ describe DarwinCore do
       file = File.join(@file_dir, 'invalid.tar.gz')
       lambda { DarwinCore.new(file) }.should raise_error(DarwinCore::InvalidArchiveError)
     end
-    
+
     it "should raise an error if archive is not in utf-8" do
       file = File.join(@file_dir, 'latin1.tar.gz')
       lambda { DarwinCore.new(file) }.should raise_error(DarwinCore::EncodingError)
@@ -219,9 +219,9 @@ describe DarwinCore do
       dwc = DarwinCore.new(file)
       norm = dwc.normalize_classification
       taxa = norm.select{|k,v| v.current_name_canonical.match " "}.select{|k,v| [v.current_name.split(" ").size >  v.current_name_canonical.split(" ").size]}
-      taxa.size.should == 2 
+      taxa.size.should == 2
     end
-    
+
     it "should be able to get language and locality fields for vernacular names" do
       file = File.join(@file_dir, 'language_locality.tar.gz')
       dwc = DarwinCore.new(file)
@@ -231,6 +231,20 @@ describe DarwinCore do
       vn.language.should == 'en'
       vn.locality.should == 'New England'
     end
+
+    it 'should be able to get uuids from gnub dataset' do
+      file = File.join(@file_dir, 'gnub.tar.gz')
+      dwc = DarwinCore.new(file)
+      cn = DarwinCore::ClassificationNormalizer.new(dwc)
+      cn.normalize
+      vn = cn.normalized_data['9c399f90-cfb8-5a7f-9a21-18285a473488']
+      vn.class.should == DarwinCore::GnubTaxon
+      vn.uuid.should == '8faa91f6-663f-4cfe-b785-0ab4e9415a51'
+      vn.uuid_path.should == [
+        "9a9f9eeb-d5f9-4ff6-b6cb-a5ad345e33c3",
+        "bf4c91c0-3d1f-44c7-9d3b-249382182a26",
+        "8faa91f6-663f-4cfe-b785-0ab4e9415a51"]
+    end
   end
 
 end

data/spec/spec_helper.rb

@@ -1,6 +1,5 @@
 $LOAD_PATH.unshift(File.dirname(__FILE__))
 $LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
-require 'rubygems'
 require 'dwc-archive'
 require 'rspec'
 require 'rspec/autorun'

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: dwc-archive
 version: !ruby/object:Gem::Version
-  version: 0.9.5
+  version: 0.9.6
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-06-17 00:00:00.000000000 Z
+date: 2013-06-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: nokogiri
@@ -171,6 +171,7 @@ files:
 - spec/files/empty_coreid.tar.gz
 - spec/files/file with characters(3).gz
 - spec/files/flat_list.tar.gz
+- spec/files/gnub.tar.gz
 - spec/files/invalid.tar.gz
 - spec/files/junk_dir_inside.zip
 - spec/files/language_locality.tar.gz
@@ -202,7 +203,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: -2036702231114584390
+      hash: 2372622078447411988
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements: