taxonifi 0.1.0 → 0.2.0

data/lib/lumper/clump.rb

@@ -0,0 +1,112 @@
+# require File.expand_path(File.join(File.dirname(__FILE__), '../taxonifi'))
+
+# A Clump is a "C"ollection of lump derivatives and the relatinoships between these derivatives!
+# It's used to define relationships among objects derived, for example, between single rows of data
+module Taxonifi::Lumper:Clumps
+
+  class Taxonifi::Lumper::Clump
+
+    attr_accessor :collections 
+    attr_accessor :annonymous_collection_index
+    attr_accessor :csv
+
+    def initialize(csv = nil)
+      @collections = {}
+      @annonymous_collection_index = 0
+      @csv = csv if !csv.nil?
+      @csv ||= nil
+    end
+
+    def add_csv(csv)
+      if @csv.nil?
+        @csv = csv
+      else
+        return false
+      end
+    end
+
+    def remove_csv
+      if !@csv.nil?
+        @csv = nil 
+        true
+      else
+        false
+      end
+    end
+
+    def get_from_csv(options = {})
+      opts = {
+        collection: :name
+      }.merge!(options) 
+      raise if @csv.nil?
+      raise if not Taxonifi::Model::Collection.subclass_prefixes.include?(opts[:collection].to_s)
+
+      case opts[:collection]
+      when :name
+        add_name_collection(opts)
+      when :ref
+        add_ref_collection(opts)
+      else
+        raise
+      end
+    end 
+
+    def next_available_collection_name
+      "collection#{annonymous_collection_index}"
+    end
+
+    def increment_annonymous_collection_index
+      @annonymous_collection_index += 1
+      true
+    end  
+
+    def add_ref_collection(options)
+      opts = {
+        :name => next_available_collection_name
+      }.merge!(options)
+      if   add_collection(opts[:name],Taxonifi::Model::RefCollection.new(opts))
+        increment_annonymous_collection_index if (opts[:name] == next_available_collection_name)
+        true
+      else
+        false
+      end
+    end
+
+    def add_name_collection(options)
+      opts = {
+        :name => next_available_collection_name
+      }.merge!(options)
+      if add_collection(opts[:name],Taxonifi::Model::NameCollection.new(opts))
+        increment_annonymous_collection_index if opts[:name] == next_available_collection_name
+        true
+      else
+        false 
+      end
+    end
+
+    def add_collection(name = nil, collection = nil)
+      return false if (name.nil? || collection.nil?)
+      return false if @collections.keys.include?(name)
+      @collections.merge!(name => collection)
+    end
+
+
+    def link(collection1, collection2, link_method)
+
+    end
+
+    def self.link_name_collection_and_ref_collection(options = {})
+      opt = {
+        :nc => Taxonifi::Model::NameCollection.new,
+        :rc => Taxonifi::Model::RefCollection.new,
+        :by => :row_number
+      }
+    end
+
+    # Should ultimately make this a reddis hook
+
+    # variable indecies b/w data
+
+
+  end 
+end

data/lib/lumper/lumper.rb

@@ -29,7 +29,7 @@ module Taxonifi::Lumper
     quad_author_year: QUAD + AUTHOR_YEAR,
     names:  Taxonifi::RANKS + AUTHOR_YEAR,
     higher: Taxonifi::RANKS - [QUAD + AUTHOR_YEAR],
-    species: ['species', 'subspecies'],
+    species: ['species', 'subspecies', 'variety'],
     genera: ['genus', 'subgenus'],
     citation_basic: %w{authors year title publication volume number pages pg_start pg_end},
     citation_small: %w{authors year title publication volume_number pages},
@@ -37,6 +37,8 @@ module Taxonifi::Lumper
     eol_basic: %w{identifier parent child rank synonyms}
   }
 
+  # Authors, Year, Title, Publication, Volume_Number Pages Cited_Page
+
   # Lumps for which all columns are represented 
   # TODO: This is really an assessor method 
   def self.available_lumps(columns)
@@ -55,21 +57,26 @@ module Taxonifi::Lumper
     intersections
   end
 
+ 
   # Return a Taxonifi::Model::NameCollection from a csv file.
-  def self.create_name_collection(csv)
+  def self.create_name_collection(options = {})
+    opts = {
+      :csv => [],
+      :initial_id => 0,
+      :capture_related_fields => true   # Stores other column values in (column_header => value) pairs in Name.related
+    }.merge!(options)
+    
+    csv = opts[:csv]
     raise Taxonifi::Lumper::LumperError, 'Something that is not a CSV::Table was passed to Lumper.create_name_collection.' if csv.class != CSV::Table
-    nc = Taxonifi::Model::NameCollection.new
-
+    
+    nc = Taxonifi::Model::NameCollection.new(:initial_id => opts[:initial_id])
     row_size = csv.size
 
     # The row index contains a vector of parent ids like
     # [0, 4, 29]
     # This implies that Name with #id 29 has Parent with #id 4
     # Initialize an empty index. 
-    row_index = []
-    (0..(row_size-1)).each do |i|
-      row_index[i] = []
-    end
+    row_index = Taxonifi::Utils::Array.build_array_of_empty_arrays(row_size)
 
     # The name_index keeps track of unique name per rank like
     # :genus => {'Foo' => [0,2]}
@@ -77,27 +84,30 @@ module Taxonifi::Lumper
     # name collection, with id 0, and id 2.
     name_index = {} 
 
+    has_ref_fields = ([:citation_basic, :citation_small] & Taxonifi::Lumper.intersecting_lumps(csv.headers)).size > 0
+    unused_fields = csv.headers - Taxonifi::Lumper::LUMPS[:names]
+
+
     # First pass, create and index names
     Taxonifi::Assessor::RowAssessor.rank_headers(csv.headers).each do |rank|
       name_index[rank] = {}
       csv.each_with_index do |row, i|
-        row_rank = Taxonifi::Assessor::RowAssessor.lump_name_rank(row).to_s # metadata (e.g. author year) apply to this rank 
-
+        shares_rank = (rank == Taxonifi::Assessor::RowAssessor.lump_name_rank(row).to_s)
         name = row[rank] 
 
         if !name.nil?     # cell has data
           n = nil         # a Name if necessary
-          name_id = nil   # index the new or existing name 
+          name_id = nil   # index the new or existing Name 
 
-          if name_index[rank][name] # name (string) exists
+          if name_index[rank][name] # A matching name (String) has been previously added
+            exists = false
 
-            exists = false 
             name_index[rank][name].each do |id|
               # Compare vectors of parent_ids for name presence
               if nc.parent_id_vector(id) == row_index[i]      
                 exists = true
                 name_id = id
-                break # don't need to check further
+                break 
               end 
             end
 
@@ -109,7 +119,8 @@ module Taxonifi::Lumper
             n = Taxonifi::Model::Name.new()
           end # end name exists
 
-          # If we created a new name
+
+          # Populate the new name if created.  Previously matched names are not effected. 
           if !n.nil? 
             n.rank = rank
             n.name = name
@@ -119,17 +130,26 @@ module Taxonifi::Lumper
             # Name/year needs to be standardized / cased out
             # headers are overlapping at times
 
-            if row['author_year'] && row_rank == rank
-              builder = Taxonifi::Splitter::Builder.build_author_year(row['author_year'])                
-              n.author               = builder.people 
-              n.year                 = builder.year 
-              n.parens               = !builder.parens
+            # Check to see if metadata (e.g. author year) apply to this rank, attach if so.
+            if shares_rank 
+              if row['author_year'] 
+                builder = Taxonifi::Splitter::Builder.build_author_year(row['author_year'])                
+                n.author               = builder.people 
+                n.year                 = builder.year 
+                n.parens               = !builder.parens
+              end
+
+              n.related.merge!(:link_to_ref_from_row => i) if has_ref_fields
+              n.related.merge!(row.to_hash.select{|f| unused_fields.include?(f)}) if opts[:capture_related_fields]
             end
 
             name_id = nc.add_object(n).id
-            # Add the name to the index of unique names
             name_index[rank][name] ||= []
             name_index[rank][name].push name_id                
+
+            $DEBUG && $stderr.puts("added #{nc.collection.size - 1} | #{n.name} | #{n.rank} | #{n.parent ? n.parent.name : '-'} | #{n.parent ? n.parent.id : '-'}")
+          else
+            $DEBUG && $stderr.puts("already present #{rank} | #{name}")
           end
 
           # build a by row vector of parent child relationships
@@ -138,15 +158,22 @@ module Taxonifi::Lumper
 
       end
     end
-
     nc
   end 
 
   # Return a Taxonifi::Model::RefCollection from a CSV file.
-  def self.create_ref_collection(csv)
+  def self.create_ref_collection(options = {})
+    opts = {
+      :csv => nil,
+      :inital_id => 1,
+      :capture_related_fields => true   # Stores other column values in (column_header => value) pairs in Ref.related
+    }.merge!(options)
+    csv = opts[:csv]
+
     raise Taxonifi::Lumper::LumperError, 'Something that is not a CSV::Table was passed to Lumper.create_ref_collection.' if csv.class != CSV::Table
-    rc = Taxonifi::Model::RefCollection.new
-    row_size = csv.size
+    rc = Taxonifi::Model::RefCollection.new(opts)
+
+    unused_fields = csv.headers - (Taxonifi::Lumper::LUMPS[:citation_basic] | Taxonifi::Lumper::LUMPS[:citation_small])
 
     ref_index = {}
     csv.each_with_index do |row, i|
@@ -178,30 +205,41 @@ module Taxonifi::Lumper
 
         if row['pages'] && !row['pages'].empty?
           # If our regex doesn't match dump the field into pages
+          lexer = Taxonifi::Splitter::Lexer.new(row['pages'], :pages)
           begin
-            lexer = Taxonifi::Splitter::Lexer.new(row['pages'], :pages)
-            t = lexer.pop(Taxonifi::Splitter::Tokens::Pages)
+          if t = lexer.pop(Taxonifi::Splitter::Tokens::Pages)
             r.pg_start = t.pg_start
             r.pg_end = t.pg_end
+            r.pages = t.remainder
+          else
+            r.pages = row['pages']
+          end
           rescue
             r.pages = row['pages']
           end
         end
+       
+        r.related.merge!(row.to_hash.select{|f| unused_fields.include?(f)}) if opts[:capture_related_fields]
 
         # Do some indexing.
         ref_str = r.compact_string 
         if !ref_index.keys.include?(ref_str)
           ref_id = rc.add_object(r).id
           ref_index.merge!(ref_str => ref_id)
+          # puts "#{i} : #{ref_id}"
           rc.row_index[i] = r 
         else
-          rc.row_index[i] = ref_index[ref_str] 
+          rc.row_index[i] = rc.object_by_id(ref_index[ref_str])
+          # puts "#{i} : #{ref_index[ref_str]}"
         end
       end
     end
     rc
   end
 
+  # def self.link_name_and_ref_collections_by_row(nc, rc)
+  # end
+
   # Creates a generic Collection with Objects of GenericObject
   # Objects are assigned to parents (rank) according to the order provided in headers.
   # Objects are considered the same if they have the same name and the same parents closure, e.g.
@@ -223,15 +261,8 @@ module Taxonifi::Lumper
     row_size = csv.size
 
     # See create_name_collection
-    row_index = []
-    (0..(row_size-1)).each do |i|
-      row_index[i] = []
-    end
-
-    name_index = {}
-    headers.each do |h|
-      name_index[h] = {}
-    end
+    row_index = Taxonifi::Utils::Array.build_array_of_empty_arrays(row_size)
+    name_index = Taxonifi::Utils::Hash.build_hash_of_hashes_with_keys(headers)
 
     csv.each_with_index do |row, i|
       headers.each do |rank|
@@ -240,7 +271,7 @@ module Taxonifi::Lumper
           o = nil                      # a Name if necessary
           name_id = nil                # index the new or existing name 
 
-          if name_index[rank][name] # name exists
+          if name_index[rank][name] # Matching name is found 
 
             exists = false
             name_index[rank][name].each do |id|
@@ -267,8 +298,8 @@ module Taxonifi::Lumper
             name_id = c.add_object(o).id 
             name_index[rank][name] ||= []
             name_index[rank][name].push name_id                
-
           end
+
           row_index[i].push name_id                       
         end
       end
@@ -282,12 +313,7 @@ module Taxonifi::Lumper
     gc = Taxonifi::Model::GeogCollection.new
 
     row_size = csv.size
-
-    # See create_name_collection
-    row_index = []
-    (0..(row_size-1)).each do |i|
-      row_index[i] = []
-    end
+    row_index = Taxonifi::Utils::Array.build_array_of_empty_arrays(row_size)
 
     name_index = {}
     geog_headers =  Taxonifi::Assessor::RowAssessor.geog_headers(csv.headers)

data/lib/lumper/lumps/parent_child_name_collection.rb

@@ -8,7 +8,12 @@ module Taxonifi::Lumper::Lumps::ParentChildNameCollection
     raise Taxonifi::Lumper::LumperError, "CSV does not have the required headers (#{Taxonifi::Lumper::LUMPS[:eol_basic].join(", ")})." if  !Taxonifi::Lumper.available_lumps(csv.headers).include?(:eol_basic)
 
     nc = Taxonifi::Model::NameCollection.new(:initial_id => 1)
-    external_index = {} # identifier => Taxonifi::Name
+
+    # identifier => Taxonifi::Name
+    external_index = {} 
+
+    # Array of Hashes {:synonyms => "Name|Name1|Name2", :external_index => external_index[parent_id], :valid_species_id => valid_species_id}, {} ...
+    synonym_list = [] 
 
     csv.each_with_index do |row,i|
       name = row['child']
@@ -17,6 +22,11 @@ module Taxonifi::Lumper::Lumps::ParentChildNameCollection
       external_id = row['identifier'].to_i
       valid_species_id = nil
 
+      # Fix me
+      index_rank = 'species_group' if rank == 'species' || rank == 'subspecies'
+      index_rank = 'genus_group' if rank == 'subgenus' || rank == 'genus'
+      index_rank ||= rank
+
       case rank
       when 'species', nil
        valid_species_id = add_species_names_from_string(nc, name, external_index[parent_id])
@@ -24,10 +34,10 @@ module Taxonifi::Lumper::Lumps::ParentChildNameCollection
       else  # Just a single string, we don't have to break anything down.
         n = nil
 
-        if nc.by_name_index[rank][name]
+        if nc.by_name_index[index_rank][name]
           exists = false
           # TODO: this hasn't been hit yet
-          nc.by_name_index[rank][name].each do |id|
+          nc.by_name_index[index_rank][name].each do |id|
             if nc.parent_id_vector(id).pop == nc.parent_id_vector(parent_id)
               exists = true
               break
@@ -45,39 +55,93 @@ module Taxonifi::Lumper::Lumps::ParentChildNameCollection
           # TODO: No author, year have yet been observed for genus and higher names
           n.rank = rank
           n.name = name
-          n.external_id = external_id
           n.row_number = i
-    
+          n.related.merge!(:external_id => external_id)
+
           if parent = external_index[parent_id] 
             n.parent = parent
           end
 
-          nc.add_object(n)
-          external_index.merge!(external_id => n) 
+          if !nc.name_exists?(n)
+            nc.add_object(n)
+            external_index.merge!(external_id => n) 
+          end
         end
       end
 
       if !row['synonyms'].nil? && row['synonyms'].size > 0 
-        other_names = row['synonyms'].split("|")
-        other_names.each do |n|
-          add_species_names_from_string(nc, n, external_index[parent_id], valid_species_id) 
-        end
+        #  puts n.name if external_index[parent_id].nil?
+        synonym_list.push({:synonyms => row['synonyms'], :valid_species_id => valid_species_id, :external_index => external_index[parent_id]})
       end
 
     end # end row
+
+    # parse the synonyms last, because names might have been mixed
+    synonym_list.each do |s|
+      other_names = s[:synonyms].split("|")
+      other_names.each do |n|
+        # puts ":: #{n} :: #{s[:external_index]} :: #{s[:valid_species_id]}" if s[:external_index].nil?
+        add_species_names_from_string(nc, n, s[:external_index], s[:valid_species_id]) 
+      end
+    end
+
     nc 
   end
 
-  # Add the individual names in a species epithet string.  Assumes parents all previously created.
+  # Add the last name in a species epithet string if new, record a new combination otherwise.  
+  # Assumes ALL parents have been previously added, including those used in Synonym combinations.
+  # For example, given a row with name, synonym fields like:
+  #    'Neortholomus scolopax (Say, 1832)', 'Lygaeus scolopax Say, 1832']
+  # The names Neortholomus and Lygaeus must exist.
+  #
   def self.add_species_names_from_string(nc, string, parent = nil, synonym_id = nil)
     names = Taxonifi::Splitter::Builder.build_species_name(string) # A Taxonifi::Model::SpeciesName instance
     if !parent.nil?                                                # nc.object_by_id(parent_id)
-      names.names.last.parent = parent                             # swap out the genus to the Model referenced by parent_id 
+      names.names.last.parent = parent                             # swap out the parent with the id referenced by the parent_id 
     else
       raise Taxonifi::Lumper::LumperError, "Parent of [#{names.names.last.name}] within [#{names.display_name}] not yet instantiated. \n !! To resolve: \n\t 1) If this is not a species name your file may be missing a value in the 'Rank' column (nil values are assumed to be species, all other ranks must be populated). \n\t 2) Parent names must be read before children, check that this is the case."
     end
-    last_id = nc.add_object(names.names.last).id
-    nc.object_by_id(last_id).related_name = nc.object_by_id(synonym_id) if !synonym_id.nil?
+
+    last_id = nil
+    if !nc.name_exists?(names.names.last)
+      last_id = nc.add_object(names.names.last).id
+      nc.object_by_id(last_id).related_name = nc.object_by_id(synonym_id) if !synonym_id.nil?
+    else
+
+      tmp_genus = names.genus.clone
+      # tmp_subgenus = names.subgenus.clone if !names.subgenus.nil?
+      tmp_species = names.species.clone
+      tmp_subspecies = names.subspecies.clone if !names.subspecies.nil?
+
+      case parent.rank
+      when 'genus' 
+        tmp_genus.parent = parent.parent # OK
+      when 'subgenus'
+        tmp_genus.parent = parent.parent # OK
+      when 'species'
+        tmp_genus.parent = parent.parent.parent
+        tmp_species = parent
+        tmp_subspecies.parent = tmp_species
+      end
+      
+      # tmp_subgenus.parent = tmp_genus if !tmp_subgenus.nil?
+      # real_subgenus = nc.object_by_id(nc.name_exists?(tmp_subgenus)) if !tmp_subgenus.nil? 
+
+      real_genus = nc.object_by_id(nc.name_exists?(tmp_genus)) 
+      real_species = nc.object_by_id(nc.name_exists?(tmp_species)) 
+
+      # !! Existing demo data Lygaeoidea have synonyms in which the genus name is not instantiated.  This might be a problem with DwC file 
+      # validation in general, something to look at, for now, throw up our hands and move on.
+      return last_id if (real_genus.nil? || real_species.nil?)
+
+      # debugger if real_genus.id == 399
+      real_subgenus = nil # revisit
+      real_subspecies = nc.object_by_id(nc.name_exists?(tmp_subspecies))  if !tmp_subspecies.nil?
+    
+      rc = [real_genus, real_subgenus, real_species, real_subspecies]
+      nc.combinations.push rc
+    end
+
     last_id
   end
 

data/lib/models/author_year.rb

@@ -3,7 +3,7 @@ require File.expand_path(File.join(File.dirname(__FILE__), "../models/base.rb"))
 module Taxonifi
   module Model
     # A class to aggregate People and Year combinations.
-   class AuthorYear < Taxonifi::Model::Base
+    class AuthorYear < Taxonifi::Model::Base
       # Array of Taxonifi::Model::People
       attr_accessor :people
       # String 
@@ -32,7 +32,6 @@ module Taxonifi
         end
         index.join("-")
       end
-
     end
   end
 end

data/lib/models/base.rb

@@ -1,73 +1,78 @@
 module Taxonifi
   class ModelError < StandardError; end
   module Model
-  
+
+    require File.expand_path(File.join(File.dirname(__FILE__), 'shared_class_methods'))
+
     # A base class for all Taxonifi::Models that represent
     # "individuals" (as opposed to collections of indviduals).  
     class Base 
+
+      include Taxonifi::Model::SharedClassMethods
+
       # The id of this object.
       attr_accessor :id
+
       # Optionly store the row this came from
       attr_accessor :row_number
-      # Optionally store an id representing the original id usef for this record. 
-      attr_accessor :external_id
 
-        # Assign on new() all attributes for the ATTRIBUTES 
-        # constant in a given subclass. 
-        # !! Check validity prior to building.
-        def build(attributes, opts)
-          attributes.each do |c|
-            self.send("#{c}=",opts[c]) if !opts[c].nil?
-          end
-        end
+      # A general purpose hash populable as needed for related metadata
+      attr_accessor :related 
 
-        def id=(id)
-          raise Taxonifi::ModelError, "Base model objects must have Fixnum ids." if !id.nil? && id.class != Fixnum
-          @id = id
-        end
+      # TODO: Rethink this. See @@ATTRIBUTES in subclasses.
+      ATTRIBUTES = [:row_number]
+
+      def initialize(options = {})
+        @related = {}
+      end
 
-        # The ids only of ancestors.
-        # Immediate ancestor id is in [].last
-        def ancestor_ids
-          i = 0 # check for recursion
-          ids = []
-          p = parent 
-          while !p.nil?
-            ids.unshift p.id
-            p = p.parent
-            i += 1
-            raise Taxonifi::ModelError, "Infite recursion in parent string detected for Base model object #{id}." if i > 100
-          end
-          ids
+      # Assign on new() all attributes for the ATTRIBUTES 
+      # constant in a given subclass. 
+      # !! Check validity prior to building.
+      def build(attributes, opts)
+        attributes.each do |c|
+          self.send("#{c}=",opts[c]) if !opts[c].nil?
         end
+      end
+
+      def id=(id)
+        raise Taxonifi::ModelError, "Base model objects must have Fixnum ids." if !id.nil? && id.class != Fixnum
+        @id = id
+      end
 
-        # Ancestor objects for subclasses
-        # that have a parent property.
-        # TODO: check for parent attributes
-        def ancestors
-          i = 0 # check for recursion
-          ancestors = []
-          p = parent 
-          while !p.nil?
-            ancestors.unshift p
-            p = p.parent
-            i += 1
-            raise Taxonifi::ModelError, "Infite recursion in parent string detected for Base model object #{id.display_name}." if i > 100
-          end
-          ancestors 
+      # The ids only of ancestors.
+      # Immediate ancestor id is in [].last
+      def ancestor_ids
+        i = 0 # check for recursion
+        ids = []
+        p = parent 
+        while !p.nil?
+          ids.unshift p.id
+          p = p.parent
+          i += 1
+          raise Taxonifi::ModelError, "Infite recursion in parent string detected for Base model object #{id}." if i > 100
         end
+        ids
+      end
 
-        # Determines identity base ONLY
-        # on attributes in ATTRIBUTES.
-        def identical?(obj)
-          raise Taxonifi::ModelError, "Objects are not comparible." if obj.class != self.class
-          self.class::ATTRIBUTES.each do |a|
-            next if a == :id # don't compare
-            return false if obj.send(a) != self.send(a)
-          end
-          return true
+      # Ancestor objects for subclasses
+      # that have a parent property.
+      # TODO: check for parent attributes
+      def ancestors
+        i = 0 # check for recursion
+        ancestors = []
+        p = parent 
+        while !p.nil?
+          ancestors.unshift p
+          p = p.parent
+          i += 1
+          raise Taxonifi::ModelError, "Infite recursion in parent string detected for Base model object #{id.display_name}." if i > 100
         end
+        ancestors 
+      end
 
     end
+
+
   end
 end