dwc-archive 0.5.1 → 0.5.2

data/VERSION

@@ -1 +1 @@
-0.5.1
+0.5.2

data/lib/dwc-archive.rb

@@ -52,10 +52,6 @@ class DarwinCore
     end
   end
 
-  def self.logger
-    @@logger ||= Logger.new(nil)
-  end
-
   def self.logger=(logger)
     @@logger = logger
   end
@@ -64,17 +60,22 @@ class DarwinCore
     @@logger = Logger.new(nil)
   end
 
+  def self.logger_write(obj_id, message, method = :info)
+    @@logger.send(method, "|%s|%s|" % [obj_id, message])
+  end
+
   def initialize(dwc_path, tmp_dir = DEFAULT_TMP_DIR)
+    @@logger ||= Logger.new(nil)
     @archive = DarwinCore::Archive.new(dwc_path, tmp_dir) 
-    @core = DarwinCore::Core.new(@archive)
+    @core = DarwinCore::Core.new(self)
     @metadata = DarwinCore::Metadata.new(@archive)
     @extensions = get_extensions
   end
 
   # generates a hash from a classification data with path to each node, list of synonyms and vernacular names.
-  def normalize_classification(verbose = false)
+  def normalize_classification
     return nil unless has_parent_id?
-    @classification_normalizer ||= DarwinCore::ClassificationNormalizer.new(self, verbose)
+    @classification_normalizer ||= DarwinCore::ClassificationNormalizer.new(self)
     @classification_normalizer.normalize
   end
 
@@ -89,6 +90,6 @@ class DarwinCore
     ext = @archive.meta[root_key][:extension]
     return [] unless ext
     ext = [ext] if ext.class != Array
-    ext.map { |e| DarwinCore::Extension.new(@archive, e) }
+    ext.map { |e| DarwinCore::Extension.new(self, e) }
   end
 end

data/lib/dwc-archive/classification_normalizer.rb

@@ -22,17 +22,14 @@ class DarwinCore
   class VernacularNormalized < Struct.new(:name, :language);end
 
   class ClassificationNormalizer
-    attr_accessor :verbose
     attr_reader :error_names, :tree
 
-    def initialize(dwc_instance, verbose = false)
+    def initialize(dwc_instance)
       @dwc = dwc_instance
       @core_fields = get_fields(@dwc.core)
       @extensions = @dwc.extensions.map { |e| [e, get_fields(e)] }
       @res = {}
       @parser = ParsleyStore.new(1,2)
-      @verbose = verbose
-      @verbose_count = 10000
       @name_strings = {}
       @error_names = []
       @tree = {}
@@ -47,9 +44,12 @@ class DarwinCore
     end
 
     def normalize
+      DarwinCore.logger_write(@dwc.object_id, "Started normalization of the classification")
       @res = {}
       ingest_core
+      DarwinCore.logger_write(@dwc.object_id, "Calculating the classification parent/child paths")
       calculate_classification_path
+      DarwinCore.logger_write(@dwc.object_id, "Ingesting data from extensions")
       ingest_extensions
       @res
     end
@@ -94,26 +94,23 @@ class DarwinCore
 
     def ingest_core
       raise RuntimeError, "Darwin Core core fields must contain taxon id and scientific name" unless (@core_fields[:id] && @core_fields[:scientificname])
-      puts "Reading core information" if @verbose
-      rows = @dwc.core.read[0]
-      puts "Ingesting information from the core" if @verbose
-      rows.each_with_index do |r, i|
-        count = i + 1
-        set_scientific_name(r, @core_fields)
-        puts "Ingesting %s'th record" % count if @verbose and count % @verbose_count == 0
-        #core has AcceptedNameUsageId
-        if @core_fields[:acceptednameusageid] && r[@core_fields[:acceptednameusageid]] && r[@core_fields[:acceptednameusageid]] != r[@core_fields[:id]]
-          add_synonym_from_core(@core_fields[:acceptednameusageid], r)
-        elsif !@core_fields[:acceptednameusageid] && status_synonym?(r[@core_fields[:taxonomicstatus]])
-          add_synonym_from_core(parent_id, r)
-        else
-          taxon = @res[r[@core_fields[:id]]] ? @res[r[@core_fields[:id]]] : @res[r[@core_fields[:id]]] = DarwinCore::TaxonNormalized.new
-          taxon.id = r[@core_fields[:id]]
-          taxon.current_name = r[@core_fields[:scientificname]]
-          taxon.current_name_canonical = r[@core_fields[:canonicalname]]
-          taxon.parent_id = r[parent_id] 
-          taxon.rank = r[@core_fields[:taxonrank]] if @core_fields[:taxonrank]
-          taxon.status = r[@core_fields[:taxonomicstatus]] if @core_fields[:taxonomicstatus]
+      @dwc.core.read do |rows|
+        rows[0].each do |r|
+          set_scientific_name(r, @core_fields)
+          #core has AcceptedNameUsageId
+          if @core_fields[:acceptednameusageid] && r[@core_fields[:acceptednameusageid]] && r[@core_fields[:acceptednameusageid]] != r[@core_fields[:id]]
+            add_synonym_from_core(@core_fields[:acceptednameusageid], r)
+          elsif !@core_fields[:acceptednameusageid] && status_synonym?(r[@core_fields[:taxonomicstatus]])
+            add_synonym_from_core(parent_id, r)
+          else
+            taxon = @res[r[@core_fields[:id]]] ? @res[r[@core_fields[:id]]] : @res[r[@core_fields[:id]]] = DarwinCore::TaxonNormalized.new
+            taxon.id = r[@core_fields[:id]]
+            taxon.current_name = r[@core_fields[:scientificname]]
+            taxon.current_name_canonical = r[@core_fields[:canonicalname]]
+            taxon.parent_id = r[parent_id] 
+            taxon.rank = r[@core_fields[:taxonrank]] if @core_fields[:taxonrank]
+            taxon.status = r[@core_fields[:taxonomicstatus]] if @core_fields[:taxonomicstatus]
+          end
         end
       end
     end
@@ -172,33 +169,31 @@ class DarwinCore
     end
     
     def ingest_synonyms(extension)
-      puts "Ingesting synonyms extension" if @verbose
+      DarwinCore.logger_write(@dwc.object_id, "Ingesting synonyms extension")
       ext, fields = *extension
-      ext.read[0].each_with_index do |r, i|
-        count = i + 1
-        set_scientific_name(r, fields)
-        puts "Ingesting %s'th record" % count if @verbose && count % @verbose_count == 0
-        @res[r[fields[:id]]].synonyms << SynonymNormalized.new(
-          r[fields[:scientificname]], 
-          r[fields[:canonicalname]], 
-          fields[:taxonomicstatus] ? r[fields[:taxonomicstatus]] : nil)
+      ext.read do |rows|
+        rows[0].each do |r|
+          set_scientific_name(r, fields)
+          @res[r[fields[:id]]].synonyms << SynonymNormalized.new(
+            r[fields[:scientificname]], 
+            r[fields[:canonicalname]], 
+            fields[:taxonomicstatus] ? r[fields[:taxonomicstatus]] : nil)
+        end
       end
     end
 
     def ingest_vernaculars(extension)
-      puts "Ingesting vernacular names" if @verbose
+      DarwinCore.logger_write(@dwc.object_id, "Ingesting vernacular names extension")
       ext, fields = *extension
-      ext.read[0].each_with_index do |r, i|
-        count = i + 1
-        puts "Ingesting %s'th record" % count if @verbose && count % @verbose_count == 0
-        @res[r[fields[:id]]].vernacular_names << VernacularNormalized.new(
-          r[fields[:vernacularname]],
-          fields[:languagecode] ? r[fields[:languagecode]] : nil)
-        add_name_string(r[fields[:vernacularname]])
+      ext.read do |rows|
+        rows[0].each do |r|
+          @res[r[fields[:id]]].vernacular_names << VernacularNormalized.new(
+            r[fields[:vernacularname]],
+            fields[:languagecode] ? r[fields[:languagecode]] : nil)
+          add_name_string(r[fields[:vernacularname]])
+        end
       end
     end
 
   end
 end
-
-

data/lib/dwc-archive/core.rb

@@ -2,8 +2,9 @@ class DarwinCore
   class Core
     include DarwinCore::Ingester
     attr_reader :id 
-    def initialize(archive)
-      @archive = archive
+    def initialize(dwc)
+      @dwc = dwc
+      @archive = @dwc.archive
       @path = @archive.files_path
       root_key = @archive.meta.keys[0]
       @data = @archive.meta[root_key][:core]

data/lib/dwc-archive/extension.rb

@@ -4,8 +4,9 @@ class DarwinCore
     attr_reader :coreid
     alias :id :coreid
 
-    def initialize(archive, data)
-      @archive = archive
+    def initialize(dwc, data)
+      @dwc = dwc
+      @archive = @dwc.archive
       @path = @archive.files_path
       @data = data
       @coreid = @data[:coreid][:attributes] 

data/lib/dwc-archive/ingester.rb

@@ -3,6 +3,7 @@ class DarwinCore
     attr_reader :data, :properties, :encoding, :fields_separator
     attr_reader :file_path, :fields, :line_separator, :quote_character, :ignore_headers
     def read(batch_size = 10000)
+      DarwinCore.logger_write(@dwc.object_id, "Reading %s data" % name)
       res = []
       errors = []
       index_fix = 1
@@ -13,7 +14,7 @@ class DarwinCore
         index_fix = 0; next if @ignore_headers && i == 0
         min_size > r.size ? errors << r : process_csv_row(res, errors, r)
         if (i + index_fix) % batch_size == 0
-          DarwinCore.logger.info("%s| Ingested %s records" % [self.object_id, (i + index_fix)])
+          DarwinCore.logger_write(@dwc.object_id, "Ingested %s records from %s" % [(i + index_fix), name])
           if block_given? 
             yield [res, errors]
             res = []
@@ -26,6 +27,10 @@ class DarwinCore
     end
     
     private
+    def name
+      self.class.to_s.split('::')[-1].downcase
+    end
+
     def process_csv_row(result, errors, row)
       str = row.join('')
       if R19

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: dwc-archive
 version: !ruby/object:Gem::Version 
-  hash: 9
+  hash: 15
   prerelease: false
   segments: 
   - 0
   - 5
-  - 1
-  version: 0.5.1
+  - 2
+  version: 0.5.2
 platform: ruby
 authors: 
 - Dmitry Mozzherin
@@ -15,7 +15,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-11-05 00:00:00 -04:00
+date: 2010-11-06 00:00:00 -04:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -66,9 +66,8 @@ dependencies:
   version_requirements: *id003
 description: Darwin Core Archive is the current standard exchange format for GLobal Names Architecture modules. This gem makes it easy to incorporate files in Darwin Core Archive format into a ruby project.
 email: dmozzherin at gmail dot com
-executables: 
-- preparse.rb
-- t
+executables: []
+
 extensions: []
 
 extra_rdoc_files: 
@@ -119,8 +118,6 @@ files:
 - spec/lib/ruby_extenstions_spec.rb
 - spec/spec.opts
 - spec/spec_helper.rb
-- bin/preparse.rb
-- bin/t
 has_rdoc: true
 homepage: http://github.com/GlobalNamesArchitecture/dwc-archive
 licenses: []