dwc-archive 0.7.3 → 0.7.4

data/README.rdoc

@@ -2,6 +2,9 @@
 
 Darwin Core Archive format is a current standard for information exchange between Global Names Architecture modules. This gem allows to work with Darwin Core Archive data compressed to either zip or tar.gz files. More information about Darwing Core Archive can be found on a GBIF page[http://www.gbif.org/informatics/standards-and-tools/publishing-data/data-standards/darwin-core-archives/]:
 
+
+{<img src="http://travis-ci.org/dimus/dwc-archive.png" />}[http://travis-ci.org/dwc-archive/dwc-archive]
+
 == Installation
 
 Update to latest rubygems (v >= 1.3.6) which adds gemcutter sources by default.

data/VERSION

@@ -1 +1 @@
-0.7.3
+0.7.4

data/features/dwca-reader.feature

@@ -70,3 +70,11 @@ Feature: Reading of a Darwing Core Archive
     And there are paths, synonyms and vernacular names in normalized classification
     And names used in classification can be accessed by "name_strings" method
     And nodes_ids organized in trees can be accessed by "tree" method
+  
+  Scenario: Normalizing classification without canonical names
+    Given path to a dwc file "data.tar.gz"
+    When I create a new DarwinCore instance
+    Then I am able to use DarwinCore#normalize_classification method
+    Then I am able to use normalize method without canonical names path
+    And get normalized classification in expected format
+    And there are id paths, no canonical names paths in normalized classification

data/features/step_definitions/dwc-reader_steps.rb

@@ -34,6 +34,10 @@ When /^I create a new DarwinCore instance$/ do
   end
 end
 
+When /^I create DarwinCore::ClassificationNormalizer instance$/ do
+  @cn = DarwinCore::ClassificationNormalizer.new(@dwc)
+end
+
 Then /^instance should have a valid archive$/ do
   @dwc.archive.valid?.should be_true
 end
@@ -201,6 +205,22 @@ Then /^there are paths, synonyms and vernacular names in normalized classificati
   @synonyms_are_generated.should be_true
 end
 
+Then /^there are id paths, no canonical names paths in normalized classification$/ do
+  id_paths_generated = false
+  canonical_paths_generated = false
+  @cn.normalized_data.should_not be_empty
+  @cn.normalized_data.each do |k, v|
+    if v.classification_path.size > 0
+      canonical_paths_generated = true
+    end
+    if v.classification_path_id.size > 0
+      id_paths_generated = true
+    end
+  end
+  id_paths_generated.should be_true
+  canonical_paths_generated.should be_false
+end
+
 Then /^names used in classification can be accessed by "([^"]*)" method$/ do |name_strings|
   names = @cn.send(name_strings.to_sym)
   names.size.should > @normalized_classification.size
@@ -212,7 +232,6 @@ Then /^nodes_ids organized in trees can be accessed by "([^"]*)" method$/ do |tr
     data.each do |k, v|
       keys << k
       if v != {}
-        debugger if v.class != Hash
         flatten_tree(v, keys)
       end
     end
@@ -224,3 +243,7 @@ Then /^nodes_ids organized in trees can be accessed by "([^"]*)" method$/ do |tr
   @normalized_classification.size.should == keys.size
 end
 
+Then /^I am able to use normalize method without canonical names path$/ do
+  @cn = DarwinCore::ClassificationNormalizer.new(@dwc)
+  @cn.normalize(:with_canonical_names => false)
+end

data/lib/dwc-archive/classification_normalizer.rb

@@ -44,12 +44,12 @@ class DarwinCore
       @name_strings.keys
     end
 
-    def normalize
+    def normalize(opts = {:with_canoical_names => true})
+      @with_canonical_names = opts[:with_canonical_names] != nil ? opts[:with_canonical_names] : true
       DarwinCore.logger_write(@dwc.object_id, "Started normalization of the classification")
-      @normalized_data = {}
       ingest_core
       DarwinCore.logger_write(@dwc.object_id, "Calculating the classification parent/child paths")
-      calculate_classification_path
+      has_parent_id? ? calculate_classification_path : @normalized_data.keys.each { |id| @tree[id] = {} }
       DarwinCore.logger_write(@dwc.object_id, "Ingesting data from extensions")
       ingest_extensions
       @normalized_data
@@ -59,8 +59,12 @@ class DarwinCore
 
     def get_canonical_name(a_scientific_name)
       a_scientific_name.force_encoding('utf-8')
-      canonical_name = @parser.parse(a_scientific_name, :canonical_only => true)
-      canonical_name.to_s.empty? ? a_scientific_name : canonical_name
+      if @with_canonical_names
+        canonical_name = @parser.parse(a_scientific_name, :canonical_only => true)
+        canonical_name.to_s.empty? ? a_scientific_name : canonical_name
+      else
+        nil
+      end
     end
     
     def get_fields(element)
@@ -88,13 +92,13 @@ class DarwinCore
 
     def set_scientific_name(row, fields)
       row[fields[:scientificname]] = 'N/A' unless row[fields[:scientificname]]
-      canonical_name = ''
+      canonical_name = nil
       scientific_name = row[fields[:scientificname]].strip.force_encoding('utf-8')
       if separate_canonical_and_authorship?(row, fields)
-        canonical_name = row[fields[:scientificname]].strip.force_encoding('utf-8')
+        canonical_name = row[fields[:scientificname]].strip.force_encoding('utf-8') if @with_canonical_names
         scientific_name += " #{row[fields[:scientificnameauthorship]].strip.force_encoding('utf-8')}"
       else
-        canonical_name = get_canonical_name(row[fields[:scientificname]])
+        canonical_name = get_canonical_name(row[fields[:scientificname]]) if @with_canonical_names
       end
       fields[:canonicalname] = row.size
       row << canonical_name
@@ -112,30 +116,38 @@ class DarwinCore
 
 
     def ingest_core
+      @normalized_data = {}
       raise RuntimeError, "Darwin Core core fields must contain taxon id and scientific name" unless (@core_fields[:id] && @core_fields[:scientificname])
       @dwc.core.read do |rows|
+        rows[1].each do |error|
+          @error_names << { :data => error, :error => :reading_or_encoding_error }
+        end
         rows[0].each do |r|
           set_scientific_name(r, @core_fields)
           #core has AcceptedNameUsageId
           if @core_fields[:acceptednameusageid] && r[@core_fields[:acceptednameusageid]] && r[@core_fields[:acceptednameusageid]] != r[@core_fields[:id]]
             add_synonym_from_core(@core_fields[:acceptednameusageid], r)
           elsif !@core_fields[:acceptednameusageid] && @core_fields[:taxonomicstatus] && status_synonym?(r[@core_fields[:taxonomicstatus]])
-            add_synonym_from_core(parent_id, r)
+            add_synonym_from_core(parent_id, r) if has_parent_id?
           else
             taxon = @normalized_data[r[@core_fields[:id]]] ? @normalized_data[r[@core_fields[:id]]] : @normalized_data[r[@core_fields[:id]]] = DarwinCore::TaxonNormalized.new
             taxon.id = r[@core_fields[:id]]
             taxon.current_name = r[@core_fields[:scientificname]]
             taxon.current_name_canonical = r[@core_fields[:canonicalname]]
-            taxon.parent_id = r[parent_id] 
+            taxon.parent_id = has_parent_id? ? r[parent_id] : nil 
             taxon.rank = r[@core_fields[:taxonrank]] if @core_fields[:taxonrank]
             taxon.status = r[@core_fields[:taxonomicstatus]] if @core_fields[:taxonomicstatus]
             add_name_string(taxon.current_name)
-            add_name_string(taxon.current_name_canonical)
+            add_name_string(taxon.current_name_canonical) if taxon.current_name_canonical && !taxon.current_name_canonical.empty?
           end
         end
       end
     end
     
+    def has_parent_id?
+      @has_parent_id ||= @core_fields.has_key?(:highertaxonid) || @core_fields.has_key?(:parentnameusageid)
+    end
+
     def parent_id
       parent_id_field = @core_fields[:highertaxonid] || @core_fields[:parentnameusageid]
     end
@@ -143,47 +155,52 @@ class DarwinCore
     def calculate_classification_path
       @paths_num = 0
       @normalized_data.each do |taxon_id, taxon|
-        next if !taxon.classification_path.empty?
+        next if !taxon.classification_path_id.empty?
         res = get_classification_path(taxon)
         next if res == 'error'
       end
     end
 
     def get_classification_path(taxon)
-      return if !taxon.classification_path.empty?
+      return if !taxon.classification_path_id.empty?
       @paths_num += 1
       DarwinCore.logger_write(@dwc.object_id, "Calculated %s paths" % @paths_num) if @paths_num % 10000 == 0
       current_node = {taxon.id => {}}
       if DarwinCore.nil_field?(taxon.parent_id)
-        taxon.classification_path << taxon.current_name_canonical
+        taxon.classification_path << taxon.current_name_canonical if @with_canonical_names
         taxon.classification_path_id << taxon.id
         @tree.merge!(current_node)
       else
-        parent_cp = nil
+        parent_cp = parent_cpid = nil
         if @normalized_data[taxon.parent_id]
-          parent_cp = @normalized_data[taxon.parent_id].classification_path
+          parent_cp = @normalized_data[taxon.parent_id].classification_path if @with_canonical_names
+          parent_cpid = @normalized_data[taxon.parent_id].classification_path_id 
         else
           current_parent = @normalized_data[@synonyms[taxon.parent_id]]
           if current_parent
             error = "WARNING: The parent of the taxon \'#{taxon.current_name}\' is deprecated"
-            @error_names << {:name => taxon, :error => :deprecated_parent, :current_parent => current_parent }
-            parent_cp = current_parent.classification_path
+            @error_names << {:data => taxon, :error => :deprecated_parent, :current_parent => current_parent }
+
+            parent_cp = current_parent.classification_path if @with_canonical_names
+            parent_cpid = current_parent.classification_path_id
           else
             error = "WARNING: The parent of the taxon \'#{taxon.current_name}\' not found"
-            @error_names << {:name => taxon, :error => :deprecated_parent, :current_parent => nil}
+            @error_names << {:data => taxon, :error => :deprecated_parent, :current_parent => nil}
           end  
         end
-        return 'error' unless parent_cp
-        if parent_cp.empty?
+        return 'error' unless parent_cpid
+        if parent_cpid.empty?
           res = get_classification_path(@normalized_data[taxon.parent_id]) 
           return res if res == 'error'
-          taxon.classification_path += @normalized_data[taxon.parent_id].classification_path + [taxon.current_name_canonical]
+          if @with_canonical_names
+            taxon.classification_path += @normalized_data[taxon.parent_id].classification_path + [taxon.current_name_canonical]
+          end
           taxon.classification_path_id += @normalized_data[taxon.parent_id].classification_path_id + [taxon.id]
           parent_node = @normalized_data[taxon.parent_id].classification_path_id.inject(@tree) {|node, id| node[id]}
           parent_node.merge!(current_node)
         else
-          taxon.classification_path += parent_cp + [taxon.current_name_canonical]
-          taxon.classification_path_id += @normalized_data[taxon.parent_id].classification_path_id + [taxon.id]
+          taxon.classification_path += parent_cp + [taxon.current_name_canonical] if @with_canonical_names
+          taxon.classification_path_id += parent_cpid + [taxon.id]
           parent_node = @normalized_data[taxon.parent_id].classification_path_id.inject(@tree) {|node, id| node[id]}
           begin
             parent_node.merge!(current_node)

data/spec/lib/dwc-archive_spec.rb

@@ -57,10 +57,13 @@ describe DarwinCore do
   end
 
   describe ".normalize_classification" do
-    it "should return nil if file has no parent id information" do
+    it "should return flat list if file has no parent id information" do
       file = File.join(@file_dir, 'flat_list.tar.gz')
       dwc = DarwinCore.new(file)
-      dwc.normalize_classification.should be_nil
+      cn = DarwinCore::ClassificationNormalizer.new(dwc)
+      cn.normalize
+      cn.normalized_data.should_not be_nil
+      cn.normalized_data.size.should > 0
     end
     
     it "should traverse DarwinCore files and assemble data for every node in memory" do

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: dwc-archive
 version: !ruby/object:Gem::Version
-  version: 0.7.3
+  version: 0.7.4
   prerelease: 
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2011-11-16 00:00:00.000000000Z
+date: 2011-11-28 00:00:00.000000000Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: parsley-store
-  requirement: &70195181105880 !ruby/object:Gem::Requirement
+  requirement: &70241095997220 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -21,10 +21,10 @@ dependencies:
         version: 0.3.0
   type: :runtime
   prerelease: false
-  version_requirements: *70195181105880
+  version_requirements: *70241095997220
 - !ruby/object:Gem::Dependency
   name: rspec
-  requirement: &70195181104820 !ruby/object:Gem::Requirement
+  requirement: &70241095996620 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -32,10 +32,10 @@ dependencies:
         version: 2.3.0
   type: :development
   prerelease: false
-  version_requirements: *70195181104820
+  version_requirements: *70241095996620
 - !ruby/object:Gem::Dependency
   name: nokogiri
-  requirement: &70195181104200 !ruby/object:Gem::Requirement
+  requirement: &70241095995420 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -43,10 +43,10 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *70195181104200
+  version_requirements: *70241095995420
 - !ruby/object:Gem::Dependency
   name: cucumber
-  requirement: &70195181103720 !ruby/object:Gem::Requirement
+  requirement: &70241095994720 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -54,10 +54,10 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *70195181103720
+  version_requirements: *70241095994720
 - !ruby/object:Gem::Dependency
   name: bundler
-  requirement: &70195181103100 !ruby/object:Gem::Requirement
+  requirement: &70241095993980 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -65,10 +65,10 @@ dependencies:
         version: 1.0.0
   type: :development
   prerelease: false
-  version_requirements: *70195181103100
+  version_requirements: *70241095993980
 - !ruby/object:Gem::Dependency
   name: jeweler
-  requirement: &70195181102480 !ruby/object:Gem::Requirement
+  requirement: &70241095993280 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -76,10 +76,10 @@ dependencies:
         version: 1.6.4
   type: :development
   prerelease: false
-  version_requirements: *70195181102480
+  version_requirements: *70241095993280
 - !ruby/object:Gem::Dependency
   name: ruby-debug19
-  requirement: &70195181102000 !ruby/object:Gem::Requirement
+  requirement: &70241095992540 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -87,10 +87,10 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *70195181102000
+  version_requirements: *70241095992540
 - !ruby/object:Gem::Dependency
   name: parsley-store
-  requirement: &70195181101520 !ruby/object:Gem::Requirement
+  requirement: &70241095991840 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -98,10 +98,10 @@ dependencies:
         version: 0.3.0
   type: :runtime
   prerelease: false
-  version_requirements: *70195181101520
+  version_requirements: *70241095991840
 - !ruby/object:Gem::Dependency
   name: rspec
-  requirement: &70195181101020 !ruby/object:Gem::Requirement
+  requirement: &70241095991100 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -109,10 +109,10 @@ dependencies:
         version: 1.2.9
   type: :development
   prerelease: false
-  version_requirements: *70195181101020
+  version_requirements: *70241095991100
 - !ruby/object:Gem::Dependency
   name: cucumber
-  requirement: &70195181100540 !ruby/object:Gem::Requirement
+  requirement: &70241095990400 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -120,7 +120,7 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *70195181100540
+  version_requirements: *70241095990400
 description: Darwin Core Archive is the current standard exchange format for GLobal
   Names Architecture modules. This gem makes it easy to incorporate files in Darwin
   Core Archive format into a ruby project.