synonym-finder 0.2.2

data/.document

@@ -0,0 +1,5 @@
+lib/**/*.rb
+bin/*
+- 
+features/**/*.feature
+LICENSE.txt

data/.rspec

@@ -0,0 +1 @@
+--color

data/.rvmrc

@@ -0,0 +1 @@
+rvm use ruby-1.9.2-p290@synfnd --create

data/Gemfile

@@ -0,0 +1,19 @@
+source "http://rubygems.org"
+# Add dependencies required to use your gem here.
+# Example:
+#   gem "activesupport", ">= 2.3.5"
+gem 'sqlite3'
+gem 'taxamatch_rb'
+gem 'biodiversity19'
+gem 'ruby-stemmer'
+
+# Add dependencies to develop your gem here.
+# Include everything needed to run rake, tests, features, etc.
+group :development do
+  gem "ruby-debug19"
+  gem "rspec", "~> 2.3.0"
+  gem "cucumber", ">= 0"
+  gem "bundler", "~> 1.0.0"
+  gem "jeweler", "~> 1.6.0"
+  gem "rcov", ">= 0"
+end

data/Gemfile.lock

@@ -0,0 +1,72 @@
+GEM
+  remote: http://rubygems.org/
+  specs:
+    archive-tar-minitar (0.5.2)
+    biodiversity (0.7.3)
+      json
+      treetop
+    biodiversity19 (0.7.3)
+      treetop
+    builder (3.0.0)
+    columnize (0.3.3)
+    cucumber (1.0.2)
+      builder (>= 2.1.2)
+      diff-lcs (>= 1.1.2)
+      gherkin (~> 2.4.5)
+      json (>= 1.4.6)
+      term-ansicolor (>= 1.0.5)
+    diff-lcs (1.1.2)
+    gherkin (2.4.5)
+      json (>= 1.4.6)
+    git (1.2.5)
+    jeweler (1.6.4)
+      bundler (~> 1.0)
+      git (>= 1.2.5)
+      rake
+    json (1.5.3)
+    linecache19 (0.5.12)
+      ruby_core_source (>= 0.1.4)
+    polyglot (0.3.1)
+    rake (0.9.2)
+    rcov (0.9.9)
+    rspec (2.3.0)
+      rspec-core (~> 2.3.0)
+      rspec-expectations (~> 2.3.0)
+      rspec-mocks (~> 2.3.0)
+    rspec-core (2.3.1)
+    rspec-expectations (2.3.0)
+      diff-lcs (~> 1.1.2)
+    rspec-mocks (2.3.0)
+    ruby-debug-base19 (0.11.25)
+      columnize (>= 0.3.1)
+      linecache19 (>= 0.5.11)
+      ruby_core_source (>= 0.1.4)
+    ruby-debug19 (0.11.6)
+      columnize (>= 0.3.1)
+      linecache19 (>= 0.5.11)
+      ruby-debug-base19 (>= 0.11.19)
+    ruby-stemmer (0.9.1)
+    ruby_core_source (0.1.5)
+      archive-tar-minitar (>= 0.5.2)
+    sqlite3 (1.3.3)
+    taxamatch_rb (0.7.6)
+      biodiversity (~> 0.7.3)
+      biodiversity19 (~> 0.7.3)
+    term-ansicolor (1.0.5)
+    treetop (1.4.9)
+      polyglot (>= 0.3.1)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  biodiversity19
+  bundler (~> 1.0.0)
+  cucumber
+  jeweler (~> 1.6.0)
+  rcov
+  rspec (~> 2.3.0)
+  ruby-debug19
+  ruby-stemmer
+  sqlite3
+  taxamatch_rb

data/LICENSE.txt

@@ -0,0 +1,20 @@
+Copyright (c) 2011 Dmitry Mozzherin
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.rdoc

@@ -0,0 +1,64 @@
+= synonym-finder
+
+Synonym finder is a biodiversity tool for finding homotypic nomenclatural synonyms in taxonomic hierarchies.`
+
+== Installation
+
+    gem install synonym-finder
+
+== Usage
+
+   #prepare intput for the gem. It understand following array of hashes as input:
+
+   input = [
+     {id: 001, path: "Animalia|Athropoda|Insecta|Hymenoptera|Formicidae|Gnamptogenys",    name: "Gnamptogenys porcata (Emery, 1896)"},
+     {id: 003, path: "Animalia|Athropoda|Insecta|Hymenoptera|Formicidae|Gnamptogenys",    name: "Gnamptogenys triangularis (Mayr, 1887)"},
+     {id: 004, path: "Animalia|Athropoda|Insecta|Hymenoptera|Formicidae|Gnamptogenys",    name: "Gnamptogenys triangularis var. alba Brown 1992"},
+     {id: 005, path: "Animalia|Athropoda|Insecta|Hymenoptera|Formicidae|Gnamptogenys",    name: "Gnamptogenys triangularis var. borealis Brown 1992"},
+     {id: 100, path: "Animalia|Athropoda|Insecta|Hymenoptera|Formicidae|Nylanderia",      name: "Nylanderia porcata"}, #match 001, no authorhsip
+     {id: 101, path: "Animalia|Athropoda|Insecta|Hymenoptera|Formicidae|Nylanderia",      name: "Nylanderia porcatum Emery, 1896"},
+     {id: 102, path: "Animalia|Athropoda|Insecta|Hymenoptera|Formicidae|Nylanderia",      name: "Nylanderia porcatum"}...]
+
+    # please note that id can be a number of a string
+
+    requre 'synonym-finder'
+
+    sf = SynonymFinder.new(input)
+    output = sf.find_synonyms
+
+    # output should be of a following format:
+    # matched putative synonyms are collected into groups of different types
+    #
+    # [ {:type=>"chresonym", :name_ids=>[203, 204]},
+    #   {:type=>"alt_placement", :name_ids=>[400, 600]},
+    #   {:type=>"chresonym", :name_ids=>[101, 102]},
+    #   {:type=>"homotypic", :name_ids=>[203, 303]},
+    #   {:type=>"lexical_variant", :name_ids=>[800, 803]},
+    #   {:type=>"lexical_variant", :name_ids=>[801, 802]},
+    #   {:type=>"homotypic", :name_ids=>[202, 302]},
+    #   {:type=>"homotypic", :name_ids=>[1, 101]},
+    #   {:type=>"misplaced_synonym", :name_ids=>[801, 803, 802, 800]}]
+
+== Synonym types
+
+* homotypic -- possible placement of species to a different genus
+* alt_placement -- possibly the same name (i.e. genus moved to a different family
+* chresonym -- different authorship with for the same canonical form, both having the same parent
+* lexical_variant -- The same parent and genus, but species epithet suffix is different (for example change of the epithet gender)
+* misplaced_synonym -- The same parent, matching species epithet, but genus varies, usually happens if a synonyn located at the same level as species.
+
+== Contributing to synonym-finder
+
+* Check out the latest master to make sure the feature hasn't been implemented or the bug hasn't been fixed yet
+* Check out the issue tracker to make sure someone already hasn't requested it and/or contributed it
+* Fork the project
+* Start a feature/bugfix branch
+* Commit and push until you are happy with your contribution
+* Make sure to add tests for it. This is important so I don't break it in a future version unintentionally.
+* Please try not to mess with the Rakefile, version, or history. If you want to have your own version, or is otherwise necessary, that is fine, but please isolate to its own commit so I can cherry-pick around it.
+
+== Copyright
+
+Copyright (c) 2011 Dmitry Mozzherin. See LICENSE.txt for
+further details.
+

data/Rakefile

@@ -0,0 +1,52 @@
+# encoding: utf-8
+
+require 'rubygems'
+require 'bundler'
+begin
+  Bundler.setup(:default, :development)
+rescue Bundler::BundlerError => e
+  $stderr.puts e.message
+  $stderr.puts "Run `bundle install` to install missing gems"
+  exit e.status_code
+end
+require 'rake'
+
+require 'jeweler'
+Jeweler::Tasks.new do |gem|
+  # gem is a Gem::Specification... see http://docs.rubygems.org/read/chapter/20 for more options
+  gem.name = "synonym-finder"
+  gem.homepage = "http://github.com/dimus/synonym-finder"
+  gem.license = "MIT"
+  gem.summary = %Q{Synonym finder is a biodiversity tool for finding homotypic nomenclatural synonyms in taxonomic hierarchies.}
+  gem.description = %Q{Synonym finder is a biodiversity tool for finding homotypic nomenclatural synonyms in taxonomic hierarchies.}
+  gem.email = "dmozzherin@gmail.com"
+  gem.authors = ["Dmitry Mozzherin"]
+  # dependencies defined in Gemfile
+end
+Jeweler::RubygemsDotOrgTasks.new
+
+require 'rspec/core'
+require 'rspec/core/rake_task'
+RSpec::Core::RakeTask.new(:spec) do |spec|
+  spec.pattern = FileList['spec/**/*_spec.rb']
+end
+
+RSpec::Core::RakeTask.new(:rcov) do |spec|
+  spec.pattern = 'spec/**/*_spec.rb'
+  spec.rcov = true
+end
+
+require 'cucumber/rake/task'
+Cucumber::Rake::Task.new(:features)
+
+task :default => :spec
+
+require 'rake/rdoctask'
+Rake::RDocTask.new do |rdoc|
+  version = File.exist?('VERSION') ? File.read('VERSION') : ""
+
+  rdoc.rdoc_dir = 'rdoc'
+  rdoc.title = "synonym-finder #{version}"
+  rdoc.rdoc_files.include('README*')
+  rdoc.rdoc_files.include('lib/**/*.rb')
+end

data/VERSION

@@ -0,0 +1 @@
+0.2.2

data/features/support/env.rb

@@ -0,0 +1,13 @@
+require 'bundler'
+begin
+  Bundler.setup(:default, :development)
+rescue Bundler::BundlerError => e
+  $stderr.puts e.message
+  $stderr.puts "Run `bundle install` to install missing gems"
+  exit e.status_code
+end
+
+$LOAD_PATH.unshift(File.dirname(__FILE__) + '/../../lib')
+require 'synonym-finder'
+
+require 'rspec/expectations'

data/features/synonym-finder.feature

@@ -0,0 +1,9 @@
+Feature: something something
+  In order to something something
+  A user something something
+  something something something
+
+  Scenario: something something
+    Given inspiration
+    When I create a sweet new gem
+    Then everyone should see how awesome I am

data/lib/synonym-finder.rb

@@ -0,0 +1,148 @@
+require 'logger'
+require 'json'
+require 'sqlite3'
+require 'taxamatch_rb'
+require 'lingua/stemmer'
+
+Dir["#{File.dirname(__FILE__)}/synonym-finder/**/*.rb"].each {|f| require f}
+
+class SynonymFinder
+  NO_AUTH_INFO = 10
+  PARTIAL_AUTH_INFO = 20
+  AUTH_MATCH = 100
+  AUTH_NO_MATCH = 0
+
+  attr :input, :db, :matches, :part_matches
+
+  def self.logger
+    @@logger ||= Logger.new(nil)
+  end
+
+  def self.logger=(logger)
+    @@logger = logger
+  end
+
+  def self.logger_reset
+    self.logger = Logger.new(nil)
+  end
+
+  def self.logger_write(obj_id, message, method = :info)
+    self.logger.send(method, "|%s|%s|" % [obj_id, message])
+  end
+
+
+  def initialize(input, in_memory = true)
+    @input = input
+    @atomizer = Taxamatch::Atomizer.new
+    @tm = Taxamatch::Base.new
+    @stemmer = Lingua::Stemmer.new(:language => "latin")
+    @db = init_db(in_memory)
+    #tmp_populate
+    build_tree unless @db.execute("select count(*) from names")[0][0].to_i > 0
+    @matches = {}
+    @part_matches = {}
+    @duplicate_finder = DuplicateFinder.new(self)
+    @group_organizer = GroupOrganizer.new(self)
+  end
+
+  def find_matches(threshold = 5)
+    @duplicate_finder.canonical_duplicates
+    matches = @duplicate_finder.species_epithet_duplicates(threshold)
+    matches = compare_authorship(matches)
+    clean_up(matches)
+    @group_organizer.organize
+  end
+
+  private
+
+  def clean_up(matches)
+    matches.each do |key, value|
+      next if value[:type] != :chresonym && value[:auth_match] < 20
+      value[:auth_match] == 100 || value[:type] == :chresonym ? @matches[key] = value : @part_matches[key] = value
+    end
+  end
+
+  def compare_authorship(matches)
+    SynonymFinder.logger_write(self.object_id, "Matching authorship")
+    count = 0
+    matches.each do |key, value|
+      count += 1
+      SynonymFinder.logger_write(self.object_id, "Matching authors %s" % count) if count % 1000 == 0
+      ids = key.join(",")
+      res = @db.execute("select authors, years from names where id in (#{ids})")
+      data1 = {:all_authors => Marshal.load(res[0][0]), :all_years =>Marshal.load(res[0][1])}
+      data2 = {:all_authors => Marshal.load(res[1][0]), :all_years =>Marshal.load(res[1][1])}
+      if (data1[:all_authors] + data1[:all_years] + data2[:all_authors] + data2 [:all_years]) == []
+        value[:auth_match] = NO_AUTH_INFO
+      elsif (data1[:all_authors] + data1[:all_years]).empty? || (data2[:all_authors] + data2[:all_years]).empty?
+        value[:auth_match] = PARTIAL_AUTH_INFO
+      else
+        value[:auth_match] = @tm.match_authors(data1, data2) == 0 ? AUTH_NO_MATCH : AUTH_MATCH
+      end
+    end
+    matches
+  end
+
+  def build_tree
+    SynonymFinder.logger_write(self.object_id, "Ingesting data")
+    @input.each_with_index do |row, i|
+      i += 1
+      SynonymFinder.logger_write(self.object_id, "Ingesting record %s" % i) if i % 1000 == 0
+      atomized_name = @atomizer.parse row[:name] rescue nil
+      next unless atomized_name && atomized_name[:species]
+      species_string = get_species(atomized_name)
+      canonical_name = atomized_name[:genus][:string] + " " + species_string
+      @db.execute("insert into names (id, name, authors, years) values (?, ?, ?, ?)", [row[:id], row[:name], Marshal.dump(atomized_name[:all_authors]), Marshal.dump(atomized_name[:all_years])])
+      @db.execute("insert into name_parts (name_id, path, canonical, epithet, epithet_stem) values (?, ?, ?, ?, ?)", [row[:id], row[:path], canonical_name, species_string, stem_epithet(species_string)])
+    end
+  end
+
+  def init_db(in_memory)
+    if in_memory == true
+      db = SQLite3::Database.new( ":memory:" )
+      create_tables(db)
+    else
+      db_file = "/tmp/syn_finder.sql"
+      db_exist = File.exist?(db_file)
+      db = SQLite3::Database.new("/tmp/syn_finder.sql")
+      unless db_exist
+        create_tables(db)
+      end
+    end
+    db
+  end
+
+  def create_tables(db)
+    db.execute("create table names (id string primary key, name string, authors, years)")
+    # db.execute("create table paths (id integer primary key autoincrement, path)")
+    # db.execute("create table paths_names (path_id integer, name_id string, level integer, primary key (path_id, name_id))")
+    db.execute("create table name_parts (name_id string, path string, canonical string, epithet string, epithet_stem string)")
+    db.execute("create index idx_name_parts_1 on name_parts (canonical)")
+    db.execute("create index idx_name_parts_2 on name_parts (epithet_stem)")
+    db.execute("create table groups (id integer primary key, type)")
+    db.execute("create table names_groups (name_id integer, group_id integer, score_max integer, score_sum integer, score_num integer, primary key (name_id, group_id))")
+    db.execute("create index idx_names_groups_2 on names_groups (group_id)")
+  end
+
+  def get_species(atomized_name)
+    species = [atomized_name[:species][:string]]
+    species += atomized_name[:infraspecies].map {|i| i[:string]} if atomized_name[:infraspecies]
+    species.join(" ")
+  end
+
+  def stem_epithet(epithet)
+    epithet.split(" ").map { |e| @stemmer.stem(e) }.join(" ")
+  end
+
+  def tmp_populate
+    f = open("/tmp/dump.sql")
+    f.each_with_index do |line, i|
+      i += 1
+      puts "loading from dump line %s" % i if i % 10000 == 0
+      if line.match /INSERT/
+        @db.execute(line.strip)
+      end
+    end
+  end
+
+end

data/lib/synonym-finder/duplicate_finder.rb

@@ -0,0 +1,87 @@
+class SynonymFinder
+  class DuplicateFinder
+
+    def initialize(synonym_finder)
+      @synonym_finder = synonym_finder
+      @db = @synonym_finder.db
+      @matches = {}
+    end
+
+    def canonical_duplicates
+      SynonymFinder.logger_write(@synonym_finder.object_id, "Processing canonical forms")
+      @db.execute("select canonical from name_parts group by canonical having count(*) > 1").each_with_index do |canonical, i|
+        i = i + 1
+        SynonymFinder.logger_write(@synonym_finder.object_id, "Processing canonical form candidate %s" % i) if i % 100 == 0
+        names = @db.execute("select name_id, path from name_parts where canonical = ?", canonical)
+        find_pairs(names)
+      end
+      @matches.each do |key, value|
+        if value[:total_distance] == 0
+          value[:type] = :chresonym
+        else
+          value[:type] = :alt_placement
+        end
+      end
+      @matches
+    end
+
+    def find_pairs(names, threshold = 0)
+      pairs = get_pairs(names)
+      pairs.each do |pair|
+        key = [pair[0][0], pair[1][0]]
+        total_distance = get_total_distance(pair[0][1], pair[1][1])
+        value = {:total_distance => total_distance}
+        @matches[key] = value if !@matches.has_key?(key) && (threshold == 0 || total_distance <= threshold)
+      end
+    end
+      
+    def get_total_distance(path1, path2)
+      total_distance = path1.size + path2.size
+      count = 0
+      path1.zip(path2).each do |pair|
+        break if pair[0] != pair[1]
+        count += 1
+      end
+      total_distance - count * 2
+    end
+
+    def get_pairs(names)
+      names = names.map { |n| [n[0], n[1].to_s.split("|")] }
+      pairs = []
+      until names.empty?
+        name = names.pop
+        names.each {|n| pairs << [name, n].sort}
+      end
+      pairs
+    end
+
+    def species_epithet_duplicates(threshold_distance)
+      SynonymFinder.logger_write(@synonym_finder.object_id, "Processing species epithets")
+      @db.execute("select epithet_stem from name_parts group by epithet_stem having count(*) > 1").each_with_index do |stem, i|
+        i = i + 1
+        SynonymFinder.logger_write(@synonym_finder.object_id, "Processing species epithet candidate %s" % i) if i % 100 == 0
+        names = @db.execute("select name_id, path from name_parts where epithet_stem = ?", stem)
+        find_pairs(names, threshold_distance)
+      end
+      count = 0
+      SynonymFinder.logger_write(@synonym_finder.object_id, "Assigning type to found matches")
+      @matches.each do |key, value|
+        next if value.has_key?(:type)
+        count += 1
+        SynonymFinder.logger_write(@synonym_finder.object_id, "Processing match %s" % count) if count % 10000 == 0
+        if value[:total_distance] == 0
+          epithets = @db.execute("select distinct epithet from name_parts where name_id in (#{key.join(",")})")
+          if epithets.size == 1
+            value[:type] = :misplaced_synonym
+          else
+            genera = @db.execute("select canonical from name_parts where name_id in (#{key.join(",")})").map { |c| c[0].split(" ")[0] }.uniq
+            value[:type] = genera.size == 1 ? :lexical_variant : :misplaced_synonym
+          end
+        else
+          value[:type] = :homotypic
+        end
+      end
+      @matches
+    end
+  end
+end

data/lib/synonym-finder/group_organizer.rb

@@ -0,0 +1,121 @@
+class SynonymFinder
+  class GroupOrganizer
+
+    def initialize(synonym_finder)
+      @synonym_finder = synonym_finder
+      @db = @synonym_finder.db
+      @groups = {}
+    end
+
+    # Finds duplication groups for a name. A name can be one or more duplication groups: chresonym, lexical variant, homotypic, alt placement
+    def organize
+      SynonymFinder.logger_write(@synonym_finder.object_id, "Grouping results")
+      organize_matches
+      #organize_partial_matches
+      get_output
+    end
+
+    private
+
+    def organize_matches
+      @last_id = 1
+      count = 0
+      @synonym_finder.matches.each do |key, value|
+        count += 1
+        SynonymFinder.logger_write(@synonym_finder.object_id, "Grouping match %s" % count) if count % 10000 == 0
+        gr1 = get_group(key[0], value[:type])
+        gr2 = get_group(key[1], value[:type])
+        if gr1 && gr2
+          update_group(gr1, gr2) if gr1 != gr2
+          key.each { |name_id| update_score(name_id, value) }
+        elsif !gr1 && !gr2
+          create_group(key, value)
+        else
+          add_to_group(key, value)
+        end
+      end
+    end
+
+    def organize_partial_matches
+      added = {}
+      count = 0
+      @synonym_finder.part_matches.each do |key, value|
+        count += 1
+        SynonymFinder.logger_write(@synonym_finder.object_id, "Adding partial matches %s" % count) if count % 10000 == 0
+        gr1 = get_group(key[0], value[:type])
+        gr2 = get_group(key[1], value[:type])
+        if  gr1 || gr2
+          group_id, name_id, name_id_db = gr1 ? [gr1, key[1], key[0]] : [gr2, key[0], key[1]] #name without authorship
+          unless added[name_id] && added[name_id][name_id_db]
+            score = get_score(value)
+            @db.execute("insert into names_groups (name_id, group_id, score_max, score_sum, score_num) values (?, ?, ?, ?, 1)", [name_id, group_id, score, score])
+            added[name_id] = { name_id_db => 1 }
+          end
+        else
+          create_group(key, value)
+        end
+      end
+    end
+
+    def get_group(name_id, type)
+      return nil unless @groups[name_id]
+      @groups[name_id][type]
+    end
+
+    def create_group(key, value)
+      @db.execute("insert into groups (id, type) values (?, ?)", [@last_id, value[:type].to_s])
+      key.each {|i| @groups[i] = {} unless @groups.has_key?(i) }
+      score = get_score(value)
+      @groups[key[0]][value[:type]] = @groups[key[1]][value[:type]] = @last_id
+      @db.execute("insert into names_groups (name_id, group_id, score_max, score_sum, score_num) values (?, ?, ?, ?, 1)", [key[0], @last_id, score, score])
+      @db.execute("insert into names_groups (name_id, group_id, score_max, score_sum, score_num) values (?, ?, ?, ?, 1)", [key[1], @last_id, score, score])
+      @last_id += 1
+    end
+
+    def update_group(gr1, gr2)
+      @db.execute("update names_groups set group_id = ? where group_id = ?", [gr1, gr2])
+      @db.execute("delete from groups where id = ?", gr2)
+    end
+
+    def add_to_group(key, value)
+      gr1 = get_group(key[0], value[:type])
+      gr2 = get_group(key[1], value[:type])
+      name_id1, name_id2, group_id = gr1 ? [key[1], key[0], gr1] : [key[0], key[1], gr2]
+      update_score(name_id2, value)
+      score = get_score(value)
+      @groups[name_id1] = {} unless @groups.has_key?(name_id1)
+      @groups[name_id1][value[:type]] = group_id
+      @db.execute("insert into names_groups (name_id, group_id, score_max, score_sum, score_num) values (?, ?, ?, ?, 1)", [name_id1, group_id, score, score])
+    end
+
+    def update_score(name_id, value)
+      score = get_score(value)
+      group_id = get_group(name_id, value[:type])
+      @db.execute("update names_groups set score_max = max(score_max, ?),  score_sum = score_sum + ?, score_num = score_num + 1 where name_id = ? and group_id = ?", [score, score, name_id, group_id])
+    end
+
+    def get_score(value)
+      return 100 if value[:type] == :chresonym
+      return 10  if value[:alt_placement] && value[:total_length] > 8
+      score = value[:auth_match]
+    end
+
+    def get_output
+      data = @db.execute("select x.group_id, g.type, ng.name_id from (select group_id from names_groups group by group_id order by count(*), group_id) x join names_groups ng on x.group_id = ng.group_id join names n on n.id = ng.name_id join groups g on g.id = ng.group_id")
+      group = 0
+      res = []
+      current_group = nil
+      data.each do |group_id, type, name_id|
+        if group_id != group
+          res << current_group if current_group
+          group = group_id
+          current_group = { :type => type, :name_ids => [name_id] }
+        else
+          current_group[:name_ids] << name_id
+        end
+      end
+      res
+    end
+
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,19 @@
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+require 'rspec'
+require 'ostruct'
+require 'synonym-finder'
+
+# Requires supporting files with custom matchers and macros, etc,
+# in ./support/ and its subdirectories.
+Dir["#{File.dirname(__FILE__)}/support/**/*.rb"].each {|f| require f}
+
+RSpec.configure do |config|
+
+end
+
+class SynonymFinder::Spec
+  Config = OpenStruct.new(
+    :input => INPUT
+  )
+end

data/spec/support/input.rb

@@ -0,0 +1,32 @@
+class SynonymFinder::Spec
+
+  INPUT = [
+    {id: 001, path: "Animalia|Athropoda|Insecta|Hymenoptera|Formicidae|Gnamptogenys",    name: "Gnamptogenys porcata (Emery, 1896)"},
+    {id: 003, path: "Animalia|Athropoda|Insecta|Hymenoptera|Formicidae|Gnamptogenys",    name: "Gnamptogenys triangularis (Mayr, 1887)"},
+    {id: 004, path: "Animalia|Athropoda|Insecta|Hymenoptera|Formicidae|Gnamptogenys",    name: "Gnamptogenys triangularis var. alba Brown 1992"},
+    {id: 005, path: "Animalia|Athropoda|Insecta|Hymenoptera|Formicidae|Gnamptogenys",    name: "Gnamptogenys triangularis var. borealis Brown 1992"},
+    {id: 100, path: "Animalia|Athropoda|Insecta|Hymenoptera|Formicidae|Nylanderia",      name: "Nylanderia porcata"}, #match 001, no authorhsip
+    {id: 101, path: "Animalia|Athropoda|Insecta|Hymenoptera|Formicidae|Nylanderia",      name: "Nylanderia porcatum Emery, 1896"}, #match 001 by stem
+    {id: 102, path: "Animalia|Athropoda|Insecta|Hymenoptera|Formicidae|Nylanderia",      name: "Nylanderia porcatum"}, #match 001 by stem
+    {id: 200, path: "Animalia|Athropoda|Insecta|Hymenoptera|Formicidae|Brachymyrmex",    name: "Brachymyrmex obscurior Forel, 1893"},
+    {id: 201, path: "Animalia|Athropoda|Insecta|Hymenoptera|Formicidae|Brachymyrmex",    name: "Brachymyrmex brevicornis Emery, 1906"},
+    {id: 202, path: "Animalia|Athropoda|Insecta|Hymenoptera|Formicidae|Brachymyrmex",    name: "Brachymyrmex patagonicus Mayr, 1868"},
+    {id: 203, path: "Animalia|Athropoda|Insecta|Hymenoptera|Formicidae|Brachymyrmex",    name: "Brachymyrmex minutus Forel, 1893"},
+    {id: 204, path: "Animalia|Athropoda|Insecta|Hymenoptera|Formicidae|Brachymyrmex",    name: "Brachymyrmex minutus Brown, 2010"}, #chresonym match with 203
+    {id: 205, path: "Animalia|Athropoda|Insecta|Hymenoptera|Formicidae|Brachymyrmex",    name: "Brachymyrmex micropeda Forel, 1893"},
+    {id: 300, path: "Animalia|Athropoda|Insecta|Hymenoptera|Formicidae|Neobrachymyrmex", name: "Neobrachymyrmex obscurior"}, #match 200 no auth
+    {id: 301, path: "Animalia|Athropoda|Insecta|Hymenoptera|Formicidae|Neobrachymyrmex", name: "Neobrachymyrmex brevicornis"}, #match 201 no auth
+    {id: 302, path: "Animalia|Athropoda|Insecta|Hymenoptera|Formicidae|Neobrachymyrmex", name: "Neobrachymyrmex patagonicus Mayr, 1868"}, #match 203 auth
+    {id: 303, path: "Animalia|Athropoda|Insecta|Hymenoptera|Formicidae|Neobrachymyrmex", name: "Neobrachymyrmex minutus Forel"}, #match 204 no auth (part)
+    {id: 304, path: "Animalia|Athropoda|Insecta|Hymenoptera|Formicidae|Neobrachymyrmex", name: "Neobrachymyrmex micropeda Brown 1995"}, #match 205 no auth 
+    {id: 400, path: "Animalia|Athropoda|Insecta|Hymenoptera|Formicidae|Crematogaster",   name: "Crematogaster obscurata  Emery, 1895"},
+    {id: 500, path: "Animalia|Athropoda|Insecta|Hymenoptera|Tiphiidae|Diamma",           name: "Diamma obscurata  (Emery, 1895)"}, #match 2 degrees 400 auth
+    {id: 600, path: "Animalia|Athropoda|Insecta|Hymenoptera|Tiphiidae|Crematogaster",    name: "Crematogaster obscurata  Em. 1895"}, #full name match
+    {id: 700, path: "Animalia|Something1|Something2|Something3|Something4|Somename",     name: "Somename obscurata Emery"}, #distance over threshold
+    {id: 800, path: "Animalia|Athropoda|Insecta|Hymenoptera|Tiphiidae|Neobrachymyrmex",  name: "Neobrachymyrmex obscurata  (Emery, 1895)"}, #match 2 degrees 400 auth
+    {id: 801, path: "Animalia|Athropoda|Insecta|Hymenoptera|Tiphiidae|Neobrachymyrmex",  name: "Brachymyrmex obscuratum  (Emery, 1895)"}, #misplaced synonym by stem 800
+    {id: 802, path: "Animalia|Athropoda|Insecta|Hymenoptera|Tiphiidae|Neobrachymyrmex",  name: "Brachymyrmex obscurata  (Emery, 1895)"}, #misplaced synonym by epithet 800
+    {id: 803, path: "Animalia|Athropoda|Insecta|Hymenoptera|Tiphiidae|Neobrachymyrmex",  name: "Neobrachymyrmex obscuratum  (Emery, 1895)"}, #lex var by epithet 800
+  ]
+
+end

data/spec/synonym-finder_spec.rb

@@ -0,0 +1,32 @@
+require File.expand_path(File.dirname(__FILE__) + '/spec_helper')
+
+describe "SynonymFinder" do
+  before(:all) do
+    SynonymFinder.logger = Logger.new($stdout)
+    @sf = SynonymFinder.new(SynonymFinder::Spec::Config.input)
+    # @sf = SynonymFinder.new(open(File.dirname(__FILE__) + "/support/union_data.txt").read)
+  end
+
+  it "should able to ingest input in correct format" do
+    @sf.input.is_a?(Array).should be_true
+    @sf.input[0].keys.should == [:id, :path, :name]
+  end
+
+  it "should be able to find species epithet duplications" do
+    output = @sf.find_matches
+    m = @sf.matches
+    m[[1, 100]].should be_nil # 1 name has no auth
+    m[[1, 101]].should == {:total_distance=>2, :type=>:homotypic, :auth_match=>100}
+    m[[1, 102]].should be_nil # 1 name has no auth
+    m[[203, 204]].should == {:total_distance=>0, :type=>:chresonym, :auth_match=>0}
+    m[[202, 302]].should == {:total_distance=>2, :type=>:homotypic, :auth_match=>100}
+    m[[400, 500]].should == {:total_distance=>4, :type=>:homotypic, :auth_match=>100}
+    m[[400, 600]].should == {:total_distance=>4, :type=>:alt_placement, :auth_match=>100}
+    m[[400, 700]].should be_nil
+    m[[400, 800]].should == {:total_distance=>4, :type=>:homotypic, :auth_match=>100}
+    m[[800, 801]].should == {:total_distance=>0, :type=>:misplaced_synonym, :auth_match=>100}
+    m[[800, 802]].should == {:total_distance=>0, :type=>:misplaced_synonym, :auth_match=>100}
+    m[[800, 803]].should == {:total_distance=>0, :type=>:lexical_variant, :auth_match=>100}
+    output.should == [{:type=>"chresonym", :name_ids=>[203, 204]}, {:type=>"alt_placement", :name_ids=>[400, 600]}, {:type=>"chresonym", :name_ids=>[101, 102]}, {:type=>"homotypic", :name_ids=>[203, 303]}, {:type=>"lexical_variant", :name_ids=>[800, 803]}, {:type=>"lexical_variant", :name_ids=>[801, 802]}, {:type=>"homotypic", :name_ids=>[202, 302]}, {:type=>"homotypic", :name_ids=>[1, 101]}, {:type=>"misplaced_synonym", :name_ids=>[801, 803, 802, 800]}]
+  end
+end

data/synonym-finder.gemspec

@@ -0,0 +1,85 @@
+# Generated by jeweler
+# DO NOT EDIT THIS FILE DIRECTLY
+# Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
+# -*- encoding: utf-8 -*-
+
+Gem::Specification.new do |s|
+  s.name = %q{synonym-finder}
+  s.version = "0.2.2"
+
+  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
+  s.authors = [%q{Dmitry Mozzherin}]
+  s.date = %q{2011-08-12}
+  s.description = %q{Synonym finder is a biodiversity tool for finding homotypic nomenclatural synonyms in taxonomic hierarchies.}
+  s.email = %q{dmozzherin@gmail.com}
+  s.extra_rdoc_files = [
+    "LICENSE.txt",
+    "README.rdoc"
+  ]
+  s.files = [
+    ".document",
+    ".rspec",
+    ".rvmrc",
+    "Gemfile",
+    "Gemfile.lock",
+    "LICENSE.txt",
+    "README.rdoc",
+    "Rakefile",
+    "VERSION",
+    "features/step_definitions/synonym-finder_steps.rb",
+    "features/support/env.rb",
+    "features/synonym-finder.feature",
+    "lib/synonym-finder.rb",
+    "lib/synonym-finder/duplicate_finder.rb",
+    "lib/synonym-finder/group_organizer.rb",
+    "spec/spec_helper.rb",
+    "spec/support/input.rb",
+    "spec/synonym-finder_spec.rb",
+    "synonym-finder.gemspec"
+  ]
+  s.homepage = %q{http://github.com/dimus/synonym-finder}
+  s.licenses = [%q{MIT}]
+  s.require_paths = [%q{lib}]
+  s.rubygems_version = %q{1.8.6}
+  s.summary = %q{Synonym finder is a biodiversity tool for finding homotypic nomenclatural synonyms in taxonomic hierarchies.}
+
+  if s.respond_to? :specification_version then
+    s.specification_version = 3
+
+    if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
+      s.add_runtime_dependency(%q<sqlite3>, [">= 0"])
+      s.add_runtime_dependency(%q<taxamatch_rb>, [">= 0"])
+      s.add_runtime_dependency(%q<biodiversity19>, [">= 0"])
+      s.add_runtime_dependency(%q<ruby-stemmer>, [">= 0"])
+      s.add_development_dependency(%q<ruby-debug19>, [">= 0"])
+      s.add_development_dependency(%q<rspec>, ["~> 2.3.0"])
+      s.add_development_dependency(%q<cucumber>, [">= 0"])
+      s.add_development_dependency(%q<bundler>, ["~> 1.0.0"])
+      s.add_development_dependency(%q<jeweler>, ["~> 1.6.0"])
+      s.add_development_dependency(%q<rcov>, [">= 0"])
+    else
+      s.add_dependency(%q<sqlite3>, [">= 0"])
+      s.add_dependency(%q<taxamatch_rb>, [">= 0"])
+      s.add_dependency(%q<biodiversity19>, [">= 0"])
+      s.add_dependency(%q<ruby-stemmer>, [">= 0"])
+      s.add_dependency(%q<ruby-debug19>, [">= 0"])
+      s.add_dependency(%q<rspec>, ["~> 2.3.0"])
+      s.add_dependency(%q<cucumber>, [">= 0"])
+      s.add_dependency(%q<bundler>, ["~> 1.0.0"])
+      s.add_dependency(%q<jeweler>, ["~> 1.6.0"])
+      s.add_dependency(%q<rcov>, [">= 0"])
+    end
+  else
+    s.add_dependency(%q<sqlite3>, [">= 0"])
+    s.add_dependency(%q<taxamatch_rb>, [">= 0"])
+    s.add_dependency(%q<biodiversity19>, [">= 0"])
+    s.add_dependency(%q<ruby-stemmer>, [">= 0"])
+    s.add_dependency(%q<ruby-debug19>, [">= 0"])
+    s.add_dependency(%q<rspec>, ["~> 2.3.0"])
+    s.add_dependency(%q<cucumber>, [">= 0"])
+    s.add_dependency(%q<bundler>, ["~> 1.0.0"])
+    s.add_dependency(%q<jeweler>, ["~> 1.6.0"])
+    s.add_dependency(%q<rcov>, [">= 0"])
+  end
+end
+

metadata

@@ -0,0 +1,181 @@
+--- !ruby/object:Gem::Specification
+name: synonym-finder
+version: !ruby/object:Gem::Version
+  version: 0.2.2
+  prerelease: 
+platform: ruby
+authors:
+- Dmitry Mozzherin
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2011-08-12 00:00:00.000000000Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: sqlite3
+  requirement: &70213009815740 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *70213009815740
+- !ruby/object:Gem::Dependency
+  name: taxamatch_rb
+  requirement: &70213009815220 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *70213009815220
+- !ruby/object:Gem::Dependency
+  name: biodiversity19
+  requirement: &70213009814740 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *70213009814740
+- !ruby/object:Gem::Dependency
+  name: ruby-stemmer
+  requirement: &70213009814220 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *70213009814220
+- !ruby/object:Gem::Dependency
+  name: ruby-debug19
+  requirement: &70213009813740 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: *70213009813740
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: &70213009813260 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 2.3.0
+  type: :development
+  prerelease: false
+  version_requirements: *70213009813260
+- !ruby/object:Gem::Dependency
+  name: cucumber
+  requirement: &70213009812780 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: *70213009812780
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: &70213009812300 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.0.0
+  type: :development
+  prerelease: false
+  version_requirements: *70213009812300
+- !ruby/object:Gem::Dependency
+  name: jeweler
+  requirement: &70213009811820 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.6.0
+  type: :development
+  prerelease: false
+  version_requirements: *70213009811820
+- !ruby/object:Gem::Dependency
+  name: rcov
+  requirement: &70213009811340 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: *70213009811340
+description: Synonym finder is a biodiversity tool for finding homotypic nomenclatural
+  synonyms in taxonomic hierarchies.
+email: dmozzherin@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files:
+- LICENSE.txt
+- README.rdoc
+files:
+- .document
+- .rspec
+- .rvmrc
+- Gemfile
+- Gemfile.lock
+- LICENSE.txt
+- README.rdoc
+- Rakefile
+- VERSION
+- features/step_definitions/synonym-finder_steps.rb
+- features/support/env.rb
+- features/synonym-finder.feature
+- lib/synonym-finder.rb
+- lib/synonym-finder/duplicate_finder.rb
+- lib/synonym-finder/group_organizer.rb
+- spec/spec_helper.rb
+- spec/support/input.rb
+- spec/synonym-finder_spec.rb
+- synonym-finder.gemspec
+homepage: http://github.com/dimus/synonym-finder
+licenses:
+- MIT
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+      segments:
+      - 0
+      hash: 2631280896359598637
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.6
+signing_key: 
+specification_version: 3
+summary: Synonym finder is a biodiversity tool for finding homotypic nomenclatural
+  synonyms in taxonomic hierarchies.
+test_files: []