taxonifi 0.1.0

data/lib/export/format/species_file.rb

@@ -0,0 +1,341 @@
+
+module Taxonifi::Export
+
+  # Dumps tables identical to the existing structure in SpeciesFile.
+  # Will only work in the pre Identity world.  Will reconfigure
+  # as templates for Jim's work after the fact.
+  class SpeciesFile < Taxonifi::Export::Base
+
+    # tblRanks 5/17/2012
+    SPECIES_FILE_RANKS = {
+      'subspecies' =>              5,
+      'species' =>                 10,
+      'species subgroup' =>        11,      
+      'species group' =>           12,     
+      'species series' =>          14,      
+      'infragenus' =>              16,  
+      'subgenus' =>                18,
+      'genus' =>                   20,
+      'genus group' =>             22,   
+      'subtribe' =>                28,
+      'tribe' =>                   30,
+      'supertribe' =>              32,  
+      'infrafamily' =>             36,   
+      'subfamily' =>               38, 
+      'subfamily group' =>         39,       
+      'family' =>                  40,
+      'epifamily' =>               41, 
+      'superfamily' =>             42,  
+      'superfamily group' =>       44,         
+      'subinfraordinal group' =>   45,             
+      'infraorder' =>              46,  
+      'suborder' =>                8,
+      'order' =>                   50,
+      'mirorder' =>                51,
+      'superorder' =>              52,  
+      'magnorder' =>               53, 
+      'cohort' =>                  54,
+      'supercohort' =>             55,   
+      'infraclass' =>              56,  
+      'subclass' =>                58,
+      'class' =>                   60,
+      'superclass' =>              62,  
+      'infraphylum' =>             66,   
+      'subphylum' =>               68, 
+      'phylum' =>                  70,
+      'superphylum' =>             72,   
+      'infrakingdom' =>            76,   
+      'subkingdom' =>              78,  
+      'kingdom' =>                 80,
+      'superkingdom' =>            82,    
+      'life' =>                    90,
+      'unknown' =>                 100 
+    }
+
+    attr_accessor :name_collection
+    attr_accessor :ref_collection
+    attr_accessor :author_index
+    attr_accessor :genus_names, :species_names, :nomenclator
+    attr_accessor :authorized_user_id, :time
+
+    # MANIFEST order is important
+    MANIFEST = %w{tblTaxa tblRefs tblPeople tblRefAuthors tblGenusNames tblSpeciesNames tblNomenclator tblCites} 
+
+    def initialize(options = {})
+      opts = {
+        :nc => Taxonifi::Model::NameCollection.new,
+        :export_folder => 'species_file',
+        :authorized_user_id => nil
+      }.merge!(options)
+
+      super(opts)
+      raise Taxonifi::Export::ExportError, 'NameCollection not passed to SpeciesFile export.' if ! opts[:nc].class == Taxonifi::Model::NameCollection
+      raise Taxonifi::Export::ExportError, 'You must provide authorized_user_id for species_file export initialization.' if opts[:authorized_user_id].nil?
+      @name_collection = opts[:nc]
+      @authorized_user_id = opts[:authorized_user_id]
+      @author_index = {}
+     
+      # 
+      # Careful here, at present we are just generating Reference micro-citations from our names, so the indexing "just works"
+      # because it's all internal.  There will is a strong potential for key collisions if this pipeline is modified to 
+      # include references external to the initialized name_collection.  See also export_references.
+      #
+      @by_author_reference_index = {}
+      @genus_names = {}
+      @species_names = {}
+      @nomenclator = {}
+      @time = Time.now.strftime("%F %T") 
+    end 
+
+    # Export only the ref_collection. Sidesteps the main name-centric exports
+    # Note that this still uses the base @name_collection object as a starting reference,
+    # it just references @name_collection.ref_collection.  So you can do:
+    #   nc = Taxonifi::Model::NameCollection.new
+    #   nc.ref_collection = Taxonifi::Model::RefCollection.new
+    #   etc.
+    def export_references(options = {})
+      opts = {
+        :starting_ref_id => 0,
+        :starting_author_id => 0
+      }
+
+      configure_folders
+      build_author_index 
+
+      # order matters
+      ['tblPeople', 'tblRefs', 'tblRefAuthors', 'sqlRefs' ].each do |t|
+        write_file(t, send(t))
+      end
+    end
+
+    # Assumes names that are the same are the same person. 
+    def build_author_index
+      @author_index = @name_collection.ref_collection.unique_authors.inject({}){|hsh, a| hsh.merge!(a.compact_string => a)}
+    end
+      
+    def export()
+      super
+      @name_collection.generate_ref_collection(1)
+
+      # Give authors unique ids
+      @name_collection.ref_collection.uniquify_authors(1)
+      build_author_index 
+
+      # See notes in #initalize re potential key collisions!
+      @by_author_reference_index =  @name_collection.ref_collection.collection.inject({}){|hsh, r| hsh.merge!(r.author_year_index => r)}
+      
+      @name_collection.names_at_rank('genus').inject(@genus_names){|hsh, n| hsh.merge!(n.name => nil)}
+      @name_collection.names_at_rank('subgenus').inject(@genus_names){|hsh, n| hsh.merge!(n.name => nil)}
+      @name_collection.names_at_rank('species').inject(@species_names){|hsh, n| hsh.merge!(n.name => nil)}
+      @name_collection.names_at_rank('subspecies').inject(@species_names){|hsh, n| hsh.merge!(n.name => nil)}
+
+      MANIFEST.each do |f|
+        write_file(f, send(f))
+      end
+    end
+
+    def tblTaxa
+      @headers = %w{TaxonNameID TaxonNameStr RankID Name Parens AboveID RefID DataFlags AccessCode NameStatus StatusFlags OriginalGenusID LastUpdate ModifiedBy}
+      @csv_string = CSV.generate() do |csv|
+        csv << @headers  
+        @name_collection.collection.each do |n|
+          ref = @by_author_reference_index[n.author_year_index]
+          cols = {
+            TaxonNameID: n.id,
+            TaxonNameStr: n.parent_ids_sf_style,        # closure -> ends with 1 
+            RankID: SPECIES_FILE_RANKS[n.rank], 
+            Name: n.name,
+            Parens: (n.parens ? 1 : 0),
+            AboveID: (n.related_name.nil? ? (n.parent ? n.parent.id : 0) : n.related_name.id),   # !! SF folks like to pre-populate with zeros
+            RefID: (ref ? ref.id : 0),
+            DataFlags: 0,                                # see http://software.speciesfile.org/Design/TaxaTables.aspx#Taxon, a flag populated when data is reviewed, initialize to zero
+            AccessCode: 0,             
+            NameStatus: (n.related_name.nil? ? 0 : 7),                            # 0 :valid, 7: synonym)
+            StatusFlags: (n.related_name.nil? ? 0 : 262144),                      # 0 :valid, 262144: jr. synonym
+            OriginalGenusID: (!n.parens && n.parent_at_rank('genus') ? n.parent_at_rank('genus').id : 0),      # SF must be pre-configured with 0 filler (this restriction needs to go)                
+            LastUpdate: @time, 
+            ModifiedBy: @authorized_user_id,
+          }
+          csv <<  @headers.collect{|h| cols[h.to_sym]} 
+        end
+      end
+     @csv_string
+    end
+
+    # Generate a tblRefs string.
+    def tblRefs
+      @headers = %w{RefID ActualYear Title PubID  Verbatim}
+      @csv_string = CSV.generate(:col_sep => "\t") do |csv|
+        csv << @headers  
+        @name_collection.ref_collection.collection.each_with_index do |r,i|
+          cols = {
+            RefID: r.id, #  i + 1,
+            Title: (r.title.nil? ? """""" : r.title),
+            PubID: 0,                                   # Careful - assumes you have a pre-generated PubID of Zero in there, PubID table is not included in CSV imports
+            ActualYear: r.year,
+            Verbatim: r.full_citation
+          }
+          csv <<  @headers.collect{|h| cols[h.to_sym]} 
+        end
+      end
+      @csv_string
+    end
+
+    # TODO make a standard transaction wrapper
+    def sqlRefs
+      sql = [ 'BEGIN TRY', 'BEGIN TRANSACTION']
+      @headers = %w{RefID ActualYear Title PubID  Verbatim}
+      @name_collection.ref_collection.collection.each_with_index do |r,i|
+        cols = {
+          RefID: r.id, #  i + 1,
+          Title: (r.title.nil? ? """""" : r.title),
+          PubID: 0,                                   # Careful - assumes you have a pre-generated PubID of Zero in there, PubID table is not included in CSV imports
+          ActualYear: r.year,
+          Verbatim: r.full_citation
+        }
+        sql <<  "INSERT INTO tblRefs (#{@headers.sort.join(",")}) VALUES (#{@headers.sort.collect{|h| "'#{cols[h.to_sym].to_s.gsub(/'/,"''")}'"}.join(",")});"
+      end
+      sql << ['COMMIT', 'END TRY', 'BEGIN CATCH', 'ROLLBACK', 'END CATCH']
+      sql.join("\n") 
+    end
+
+    # Generate tblPeople string.
+    def tblPeople
+      @headers = %w{PersonID FamilyName GivenNames GivenInitials Suffix Role LastUpdate ModifiedBy}
+      @csv_string = CSV.generate() do |csv|
+        csv << @headers  
+        @author_index.keys.each_with_index do |k,i|
+          a = @author_index[k] 
+          # a.id = i + 1
+          cols = {
+            PersonID: a.id,
+            FamilyName: a.last_name,
+            GivenName: a.first_name,
+            GivenInitials: a.initials_string,
+            Suffix: a.suffix,
+            Role: 1,                          # authors 
+            LastUpdate: @time,
+            ModifiedBy: @authorized_user_id
+          }
+          csv <<  @headers.collect{|h| cols[h.to_sym]} 
+        end
+      end
+      @csv_string
+    end
+
+    # Generate tblRefAuthors string.
+    def tblRefAuthors 
+      @headers = %w{RefID PersonID SeqNum AuthorCount LastUpdate ModifiedBy}
+      @csv_string = CSV.generate() do |csv|
+        csv << @headers  
+        @name_collection.ref_collection.collection.each do |r| 
+          r.authors.each_with_index do |x, i|
+            a = @author_index[x.compact_string] 
+            cols = {
+              RefID: r.id,
+              PersonID: a.id,
+              SeqNum: i + 1,
+              AuthorCount: r.authors.size,
+              LastUpdate: @time,
+              ModifiedBy: @authorized_user_id
+            }
+            csv <<  @headers.collect{|h| cols[h.to_sym]} 
+          end
+        end
+      end
+      @csv_string
+    end
+
+    # Generate tblCites string.
+    def tblCites
+      @headers = %w{TaxonNameID SeqNum RefID NomenclatorID LastUpdate ModifiedBy NewNameStatus CitePages Note TypeClarification CurrentConcept ConceptChange InfoFlags InfoFlagStatus PolynomialStatus}
+      @csv_string = CSV.generate() do |csv|
+        csv << @headers  
+        @name_collection.collection.each do |n|
+          ref = @by_author_reference_index[n.author_year_index]
+          next if ref.nil?
+          cols = {
+            TaxonNameID: n.id,
+            SeqNum: 1,
+            RefID: ref.id,
+            NomenclatorID: @nomenclator[n.nomenclator_name], 
+            LastUpdate: @time, 
+            ModifiedBy: @authorized_user_id,
+            CitePages: """""",        # equates to "" in CSV speak
+            NewNameStatus: 0,
+            Note: """""",
+            TypeClarification: 0,     # We might derive more data from this
+            CurrentConcept: 1,        # Boolean, right?
+            ConceptChange: 0,         # Unspecified
+            InfoFlags: 0,             # 
+            InfoFlagStatus: 1,        # 1 => needs review
+            PolynomialStatus: 0
+          }
+          csv <<  @headers.collect{|h| cols[h.to_sym]} 
+        end
+      end
+      @csv_string
+    end
+
+    def tblGenusNames
+      @csv_string = csv_for_genus_and_species_names_tables('Genus')
+      @csv_string
+    end
+
+    def tblSpeciesNames
+      @csv_string = csv_for_genus_and_species_names_tables('Species')
+      @csv_string
+    end
+
+    def csv_for_genus_and_species_names_tables(type)
+      col = "#{type}NameID"
+      @headers = [col, "Name", "LastUpdate", "ModifiedBy", "Italicize"]
+      @csv_string = CSV.generate() do |csv|
+        csv << @headers 
+        var = self.send("#{type.downcase}_names")
+        var.keys.each_with_index do |n,i|
+          var[n] = i + 1
+          cols = {
+            col.to_sym => i + 1,
+            Name: n,
+            LastUpdate: @time, 
+            ModifiedBy: @authorized_user_id,
+            Italicize: 1                              # always true for these data
+          }
+          csv <<  @headers.collect{|h| cols[h.to_sym]} 
+        end
+      end
+      @csv_string 
+    end
+
+    # must be called post tblGenusNames and tblSpeciesNames
+    def tblNomenclator
+      @headers = %w{NomenclatorID GenusNameID SubgenusNameID SpeciesNameID SubspeciesNameID LastUpdate ModifiedBy SuitableForGenus SuitableForSpecies InfrasubspeciesNameID InfrasubKind}
+      @csv_string = CSV.generate() do |csv|
+        csv << @headers
+        i = 1
+        @name_collection.collection.each do |n|
+          next if Taxonifi::RANKS.index(n.rank) < Taxonifi::RANKS.index('genus')
+          cols = {
+            NomenclatorID: i,
+            GenusNameID: @genus_names[n.parent_name_at_rank('genus')] || 0,
+            SubgenusNameID: @genus_names[n.parent_name_at_rank('subgenus')] || 0,
+            SpeciesNameID: @species_names[n.parent_name_at_rank('species')] || 0,
+            SubspeciesNameID: @species_names[n.parent_name_at_rank('subspecies')] || 0,
+            InfrasubspeciesNameID: 0,
+            InfrasubKind: 0,                          # this might be wrong
+            LastUpdate: @time,  
+            ModifiedBy: @authorized_user_id, 
+            SuitableForGenus: 0,                      # Set in SF 
+            SuitableForSpecies: 0                     # Set in SF
+          }
+          @nomenclator.merge!(n.nomenclator_name => i)
+          i += 1
+          csv <<  @headers.collect{|h| cols[h.to_sym]} 
+        end
+      end
+      @csv_string
+    end
+
+  end
+end

data/lib/lumper/lumper.rb

@@ -0,0 +1,334 @@
+require File.expand_path(File.join(File.dirname(__FILE__), '../taxonifi'))
+
+# The lumper lumps! Tools for recognizing and using 
+# combinations of column types. 
+module Taxonifi::Lumper 
+
+  # Define groups of columns/fields and include
+  # functionality to determine whether your
+  # columns match a given set.
+  module Lumps
+    Dir.glob( File.expand_path(File.join(File.dirname(__FILE__), "lumps/*.rb") )) do |file|
+      require file
+    end
+  end
+
+  class LumperError < StandardError; end
+
+  # Columns used for species epithets.
+  # !! Todo: map DwC URIs to these labels (at present they largely correllate with Tokens,
+  # perhaps map URIs to tokens!?)
+  QUAD =  ['genus', 'subgenus', 'species', 'subspecies']
+  
+  # Columns representing author and year 
+  AUTHOR_YEAR = ['author', 'year']
+
+  # A Hash of named column combinations
+  LUMPS = {
+    quadrinomial: QUAD,
+    quad_author_year: QUAD + AUTHOR_YEAR,
+    names:  Taxonifi::RANKS + AUTHOR_YEAR,
+    higher: Taxonifi::RANKS - [QUAD + AUTHOR_YEAR],
+    species: ['species', 'subspecies'],
+    genera: ['genus', 'subgenus'],
+    citation_basic: %w{authors year title publication volume number pages pg_start pg_end},
+    citation_small: %w{authors year title publication volume_number pages},
+    basic_geog: %w{country state county}, # add 'continent'
+    eol_basic: %w{identifier parent child rank synonyms}
+  }
+
+  # Lumps for which all columns are represented 
+  # TODO: This is really an assessor method 
+  def self.available_lumps(columns)
+    raise Taxonifi::Lumper::LumperError, 'Array not passed to Lumper.available_lumps.' if !(columns.class == Array)
+    LUMPS.keys.select{|k| (LUMPS[k] - columns) == []}
+  end
+
+  # Lumps for which any column is represented 
+  # # TODO: This is really an assessor method 
+  def self.intersecting_lumps(columns)
+    raise Taxonifi::Lumper::LumperError, 'Array not passed to Lumper.intersecting_lumps.' if !(columns.class == Array)
+    intersections = []
+    LUMPS.keys.each do |k|
+      intersections.push k if (LUMPS[k] & columns).size > 0
+    end
+    intersections
+  end
+
+  # Return a Taxonifi::Model::NameCollection from a csv file.
+  def self.create_name_collection(csv)
+    raise Taxonifi::Lumper::LumperError, 'Something that is not a CSV::Table was passed to Lumper.create_name_collection.' if csv.class != CSV::Table
+    nc = Taxonifi::Model::NameCollection.new
+
+    row_size = csv.size
+
+    # The row index contains a vector of parent ids like
+    # [0, 4, 29]
+    # This implies that Name with #id 29 has Parent with #id 4
+    # Initialize an empty index. 
+    row_index = []
+    (0..(row_size-1)).each do |i|
+      row_index[i] = []
+    end
+
+    # The name_index keeps track of unique name per rank like
+    # :genus => {'Foo' => [0,2]}
+    # This says that "Foo" is instantiated two times in the
+    # name collection, with id 0, and id 2.
+    name_index = {} 
+
+    # First pass, create and index names
+    Taxonifi::Assessor::RowAssessor.rank_headers(csv.headers).each do |rank|
+      name_index[rank] = {}
+      csv.each_with_index do |row, i|
+        row_rank = Taxonifi::Assessor::RowAssessor.lump_name_rank(row).to_s # metadata (e.g. author year) apply to this rank 
+
+        name = row[rank] 
+
+        if !name.nil?     # cell has data
+          n = nil         # a Name if necessary
+          name_id = nil   # index the new or existing name 
+
+          if name_index[rank][name] # name (string) exists
+
+            exists = false 
+            name_index[rank][name].each do |id|
+              # Compare vectors of parent_ids for name presence
+              if nc.parent_id_vector(id) == row_index[i]      
+                exists = true
+                name_id = id
+                break # don't need to check further
+              end 
+            end
+
+            if !exists # name (string) exists, but parents are different, create new name 
+              n = Taxonifi::Model::Name.new()
+            end
+
+          else # no version of the name exists
+            n = Taxonifi::Model::Name.new()
+          end # end name exists
+
+          # If we created a new name
+          if !n.nil? 
+            n.rank = rank
+            n.name = name
+            n.parent = nc.object_by_id(row_index[i].last) if row_index[i].size > 0 # it's parent is the previous id in this row 
+            n.row_number = i
+
+            # Name/year needs to be standardized / cased out
+            # headers are overlapping at times
+
+            if row['author_year'] && row_rank == rank
+              builder = Taxonifi::Splitter::Builder.build_author_year(row['author_year'])                
+              n.author               = builder.people 
+              n.year                 = builder.year 
+              n.parens               = !builder.parens
+            end
+
+            name_id = nc.add_object(n).id
+            # Add the name to the index of unique names
+            name_index[rank][name] ||= []
+            name_index[rank][name].push name_id                
+          end
+
+          # build a by row vector of parent child relationships
+          row_index[i].push name_id                       
+        end # end cell has data
+
+      end
+    end
+
+    nc
+  end 
+
+  # Return a Taxonifi::Model::RefCollection from a CSV file.
+  def self.create_ref_collection(csv)
+    raise Taxonifi::Lumper::LumperError, 'Something that is not a CSV::Table was passed to Lumper.create_ref_collection.' if csv.class != CSV::Table
+    rc = Taxonifi::Model::RefCollection.new
+    row_size = csv.size
+
+    ref_index = {}
+    csv.each_with_index do |row, i|
+      if Taxonifi::Assessor::RowAssessor.intersecting_lumps_with_data(row, [:citation_small]).include?(:citation_small)
+        r = Taxonifi::Model::Ref.new(
+          :year => row['year'],
+          :title => row['title'],
+          :publication => row['publication']
+        ) 
+
+        # TODO: break out each of these lexes to a builder
+        if row['authors'] && !row['authors'].empty?
+          lexer = Taxonifi::Splitter::Lexer.new(row['authors'])
+          authors = lexer.pop(Taxonifi::Splitter::Tokens::Authors)
+          authors.names.each do |a|
+            n = Taxonifi::Model::Person.new()
+            n.last_name = a[:last_name]
+            n.initials = a[:initials]
+            r.authors.push n
+          end
+        end
+
+        if row['volume_number'] && !row['volume_number'].empty?
+          lexer = Taxonifi::Splitter::Lexer.new(row['volume_number'], :volume_number)
+          t = lexer.pop(Taxonifi::Splitter::Tokens::VolumeNumber)
+          r.volume = t.volume
+          r.number = t.number
+        end
+
+        if row['pages'] && !row['pages'].empty?
+          # If our regex doesn't match dump the field into pages
+          begin
+            lexer = Taxonifi::Splitter::Lexer.new(row['pages'], :pages)
+            t = lexer.pop(Taxonifi::Splitter::Tokens::Pages)
+            r.pg_start = t.pg_start
+            r.pg_end = t.pg_end
+          rescue
+            r.pages = row['pages']
+          end
+        end
+
+        # Do some indexing.
+        ref_str = r.compact_string 
+        if !ref_index.keys.include?(ref_str)
+          ref_id = rc.add_object(r).id
+          ref_index.merge!(ref_str => ref_id)
+          rc.row_index[i] = r 
+        else
+          rc.row_index[i] = ref_index[ref_str] 
+        end
+      end
+    end
+    rc
+  end
+
+  # Creates a generic Collection with Objects of GenericObject
+  # Objects are assigned to parents (rank) according to the order provided in headers.
+  # Objects are considered the same if they have the same name and the same parents closure, e.g.
+  #
+  #   a b c
+  #   a b d
+  #   e b f
+  #
+  #   Will return 7 objects named in order a,b,c,d,e,b,f
+  #
+  # a,b b,c b,d e,b b,f are the unique parent/child relationships stored
+  #
+  #
+  def self.create_hierarchical_collection(csv, headers)
+    raise Taxonifi::Lumper::LumperError, 'Something that is not a CSV::Table was passed to Lumper.create_name_collection.' if csv.class != CSV::Table
+    raise Taxonifi::Lumper::LumperError, 'No headers provided to create_hierarchical_collection.' if headers.size == 0
+
+    c = Taxonifi::Model::Collection.new
+    row_size = csv.size
+
+    # See create_name_collection
+    row_index = []
+    (0..(row_size-1)).each do |i|
+      row_index[i] = []
+    end
+
+    name_index = {}
+    headers.each do |h|
+      name_index[h] = {}
+    end
+
+    csv.each_with_index do |row, i|
+      headers.each do |rank|
+        name = row[rank]
+        if !name.nil? && !name.empty?  # cell has data
+          o = nil                      # a Name if necessary
+          name_id = nil                # index the new or existing name 
+
+          if name_index[rank][name] # name exists
+
+            exists = false
+            name_index[rank][name].each do |id|
+              if c.parent_id_vector(id) == row_index[i]
+                exists = true
+                name_id = id
+                break
+              end
+            end
+
+            if !exists
+              o = Taxonifi::Model::GenericObject.new()
+            end
+          else
+            o = Taxonifi::Model::GenericObject.new()
+          end
+
+          if !o.nil? 
+            o.name = name
+            o.rank = rank
+            o.row_number = i
+            o.parent = c.object_by_id(row_index[i].last) if row_index[i].size > 0 # it's parent is the previous id in this row 
+
+            name_id = c.add_object(o).id 
+            name_index[rank][name] ||= []
+            name_index[rank][name].push name_id                
+
+          end
+          row_index[i].push name_id                       
+        end
+      end
+    end
+    c
+  end
+
+  # Return a geog collection from a csv file. 
+  def self.create_geog_collection(csv)
+    raise Taxonifi::Lumper::LumperError, 'Something that is not a CSV::Table was passed to Lumper.create_geog_collection.' if csv.class != CSV::Table
+    gc = Taxonifi::Model::GeogCollection.new
+
+    row_size = csv.size
+
+    # See create_name_collection
+    row_index = []
+    (0..(row_size-1)).each do |i|
+      row_index[i] = []
+    end
+
+    name_index = {}
+    geog_headers =  Taxonifi::Assessor::RowAssessor.geog_headers(csv.headers)
+    geog_headers.each do |h|
+      name_index[h] = {}
+    end
+
+    # We don't have the same problems as with taxon names, i.e.
+    # boo in 
+    #  Foo nil boo
+    #  Foo bar boo
+    # is the same thing wrt geography, not the case for taxon names.
+    # We can use a row first loop to build as we go
+
+    csv.each_with_index do |row, i|
+      geog_headers.each do |level|
+        name = row[level]
+        if !name.nil? && !name.empty?  # cell has data
+          g = nil         # a Name if necessary
+          name_id = nil   # index the new or existing name 
+
+          if name_index[level][name] # name exists
+            name_id  = name_index[level][name] 
+          else
+            g = Taxonifi::Model::Geog.new()
+            name_id = gc.add_object(g).id
+          end
+
+          if !g.nil? 
+            g.name = name
+            g.rank = level
+            g.parent = gc.object_by_id(row_index[i].last) if row_index[i].size > 0 # it's parent is the previous id in this row 
+          end
+
+          name_index[level][name] = name_id
+          row_index[i].push name_id                       
+        end
+      end
+    end
+    gc
+  end 
+
+end # end Lumper Module 
+