taxamatch_rb 0.8.3 → 0.8.4

data/Gemfile

@@ -3,6 +3,7 @@ require 'yaml'
 # YAML::ENGINE.yamler= 'syck'
 
 gem "biodiversity19",">= 1.0.10"
+gem "damerau-levenshtein", ">= 0.5.4"
 
 
 group :development do

data/Gemfile.lock

@@ -15,6 +15,7 @@ GEM
       gherkin (~> 2.4.1)
       json (>= 1.4.6)
       term-ansicolor (>= 1.0.5)
+    damerau-levenshtein (0.5.4)
     diff-lcs (1.1.2)
     gherkin (2.4.1)
       json (>= 1.4.6)
@@ -65,6 +66,7 @@ DEPENDENCIES
   biodiversity19 (>= 1.0.10)
   bundler (~> 1.0.0)
   cucumber
+  damerau-levenshtein (>= 0.5.4)
   jeweler (~> 1.6.0)
   mocha
   rake-compiler

data/Rakefile

@@ -10,7 +10,6 @@ rescue Bundler::BundlerError => e
 end
 
 require 'rake'
-require 'rake/extensiontask'
 
 begin
   require 'jeweler'
@@ -43,10 +42,4 @@ RSpec::Core::RakeTask.new(:rcov) do |spec|
   spec.rcov = true
 end
 
-Rake::ExtensionTask.new("damerau_levenshtein") do |extension|
-    extension.lib_dir = "lib"
-end
-
-Rake::Task[:spec].prerequisites << :compile
-
 task :default => :spec

data/VERSION

@@ -1 +1 @@
-0.8.3
+0.8.4

data/lib/taxamatch_rb.rb

@@ -2,7 +2,7 @@
 $:.unshift(File.dirname(__FILE__)) unless
    $:.include?(File.dirname(__FILE__)) || $:.include?(File.expand_path(File.dirname(__FILE__)))
 # $:.unshift('taxamatch_rb')
-require 'taxamatch_rb/damerau_levenshtein_mod'
+require 'damerau-levenshtein'
 require 'taxamatch_rb/atomizer'
 require 'taxamatch_rb/normalizer'
 require 'taxamatch_rb/phonetizer'
@@ -17,7 +17,7 @@ module Taxamatch
 
     def initialize
       @parser = Taxamatch::Atomizer.new
-      @dlm = Taxamatch::DamerauLevenshteinMod.new
+      @dlm = DamerauLevenshtein
     end
 
 

data/lib/taxamatch_rb/authmatch.rb

@@ -75,7 +75,7 @@ module Taxamatch
     def self.fuzzy_match_authors(author1, author2)
       au1_length = author1.size
       au2_length = author2.size
-      dlm = Taxamatch::DamerauLevenshteinMod.new
+      dlm = DamerauLevenshtein
       ed = dlm.distance(author1, author2,2,3) #get around a bug in C code, but it really has to be fixed
       (ed <= 3 && ([au1_length, au2_length].min > ed * 2) && (ed < 2 || author1[0] == author2[0]))
     end
@@ -86,4 +86,4 @@ module Taxamatch
       nil
     end
   end
-end
+end

data/spec/taxamatch_rb_spec.rb

@@ -1,19 +1,6 @@
 # encoding: UTF-8
 require 'spec_helper'
 
-describe 'DamerauLevenshteinMod' do
-  it 'should get tests' do
-    read_test_file(File.expand_path(File.dirname(__FILE__)) + '/damerau_levenshtein_mod_test.txt', 5) do |y|
-      dl = Taxamatch::DamerauLevenshteinMod.new
-      if y
-        res = dl.distance(y[0], y[1], y[3].to_i, y[2].to_i)
-        puts y if res != y[4].to_i
-        res.should == y[4].to_i
-      end
-    end
-  end
-end
-
 describe 'Atomizer' do
   before(:all) do
     @parser = Taxamatch::Atomizer.new
@@ -54,12 +41,11 @@ describe 'Taxamatch::Base' do
   end
 
   it 'should get txt tests' do
-    dl = Taxamatch::DamerauLevenshteinMod.new
     read_test_file(File.expand_path(File.dirname(__FILE__)) + '/taxamatch_test.txt', 4) do |y|
       if y
         y[2] = y[2] == 'true' ? true : false
         res = @tm.taxamatch(y[0], y[1], false)
-        puts "%s, %s, %s, %s" % [y[0], y[1], y[2], y[3]]
+        # puts "%s, %s, %s, %s" % [y[0], y[1], y[2], y[3]]
         res['match'].should == y[2]
         res['edit_distance'].should == y[3].to_i
       end

data/taxamatch_rb.gemspec

@@ -5,14 +5,13 @@
 
 Gem::Specification.new do |s|
   s.name = "taxamatch_rb"
-  s.version = "0.8.3"
+  s.version = "0.8.4"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Dmitry Mozzherin"]
-  s.date = "2012-02-03"
+  s.date = "2012-02-13"
   s.description = "This gem implements algorithm for fuzzy matching scientific names developed by Tony Rees"
   s.email = "dmozzherin@eol.org"
-  s.extensions = ["ext/damerau_levenshtein/extconf.rb"]
   s.extra_rdoc_files = [
     "LICENSE",
     "README.rdoc"
@@ -25,14 +24,11 @@ Gem::Specification.new do |s|
     "README.rdoc",
     "Rakefile",
     "VERSION",
-    "ext/damerau_levenshtein/damerau_levenshtein.c",
     "lib/taxamatch_rb.rb",
     "lib/taxamatch_rb/atomizer.rb",
     "lib/taxamatch_rb/authmatch.rb",
-    "lib/taxamatch_rb/damerau_levenshtein_mod.rb",
     "lib/taxamatch_rb/normalizer.rb",
     "lib/taxamatch_rb/phonetizer.rb",
-    "spec/damerau_levenshtein_mod_test.txt",
     "spec/spec.opts",
     "spec/spec_helper.rb",
     "spec/taxamatch_rb_spec.rb",
@@ -49,6 +45,7 @@ Gem::Specification.new do |s|
 
     if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
       s.add_runtime_dependency(%q<biodiversity19>, [">= 1.0.10"])
+      s.add_runtime_dependency(%q<damerau-levenshtein>, [">= 0.5.4"])
       s.add_development_dependency(%q<rake-compiler>, [">= 0"])
       s.add_development_dependency(%q<rspec>, ["~> 2.3.0"])
       s.add_development_dependency(%q<cucumber>, [">= 0"])
@@ -61,6 +58,7 @@ Gem::Specification.new do |s|
       s.add_development_dependency(%q<mocha>, [">= 0"])
     else
       s.add_dependency(%q<biodiversity19>, [">= 1.0.10"])
+      s.add_dependency(%q<damerau-levenshtein>, [">= 0.5.4"])
       s.add_dependency(%q<rake-compiler>, [">= 0"])
       s.add_dependency(%q<rspec>, ["~> 2.3.0"])
       s.add_dependency(%q<cucumber>, [">= 0"])
@@ -74,6 +72,7 @@ Gem::Specification.new do |s|
     end
   else
     s.add_dependency(%q<biodiversity19>, [">= 1.0.10"])
+    s.add_dependency(%q<damerau-levenshtein>, [">= 0.5.4"])
     s.add_dependency(%q<rake-compiler>, [">= 0"])
     s.add_dependency(%q<rspec>, ["~> 2.3.0"])
     s.add_dependency(%q<cucumber>, [">= 0"])

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: taxamatch_rb
 version: !ruby/object:Gem::Version
-  version: 0.8.3
+  version: 0.8.4
   prerelease: 
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-02-03 00:00:00.000000000Z
+date: 2012-02-13 00:00:00.000000000Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: biodiversity19
-  requirement: &70185570744000 !ruby/object:Gem::Requirement
+  requirement: &70328583642780 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -21,10 +21,21 @@ dependencies:
         version: 1.0.10
   type: :runtime
   prerelease: false
-  version_requirements: *70185570744000
+  version_requirements: *70328583642780
+- !ruby/object:Gem::Dependency
+  name: damerau-levenshtein
+  requirement: &70328583642180 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 0.5.4
+  type: :runtime
+  prerelease: false
+  version_requirements: *70328583642180
 - !ruby/object:Gem::Dependency
   name: rake-compiler
-  requirement: &70185570743400 !ruby/object:Gem::Requirement
+  requirement: &70328583629080 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -32,10 +43,10 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *70185570743400
+  version_requirements: *70328583629080
 - !ruby/object:Gem::Dependency
   name: rspec
-  requirement: &70185570742800 !ruby/object:Gem::Requirement
+  requirement: &70328583628480 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -43,10 +54,10 @@ dependencies:
         version: 2.3.0
   type: :development
   prerelease: false
-  version_requirements: *70185570742800
+  version_requirements: *70328583628480
 - !ruby/object:Gem::Dependency
   name: cucumber
-  requirement: &70185570742200 !ruby/object:Gem::Requirement
+  requirement: &70328583627880 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -54,10 +65,10 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *70185570742200
+  version_requirements: *70328583627880
 - !ruby/object:Gem::Dependency
   name: bundler
-  requirement: &70185570739600 !ruby/object:Gem::Requirement
+  requirement: &70328583627280 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -65,10 +76,10 @@ dependencies:
         version: 1.0.0
   type: :development
   prerelease: false
-  version_requirements: *70185570739600
+  version_requirements: *70328583627280
 - !ruby/object:Gem::Dependency
   name: jeweler
-  requirement: &70185570739080 !ruby/object:Gem::Requirement
+  requirement: &70328583626700 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -76,10 +87,10 @@ dependencies:
         version: 1.6.0
   type: :development
   prerelease: false
-  version_requirements: *70185570739080
+  version_requirements: *70328583626700
 - !ruby/object:Gem::Dependency
   name: rcov
-  requirement: &70185570738600 !ruby/object:Gem::Requirement
+  requirement: &70328583626100 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -87,10 +98,10 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *70185570738600
+  version_requirements: *70328583626100
 - !ruby/object:Gem::Dependency
   name: ruby-debug19
-  requirement: &70185570738080 !ruby/object:Gem::Requirement
+  requirement: &70328583625500 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -98,10 +109,10 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *70185570738080
+  version_requirements: *70328583625500
 - !ruby/object:Gem::Dependency
   name: ruby-prof
-  requirement: &70185570737580 !ruby/object:Gem::Requirement
+  requirement: &70328583624960 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -109,10 +120,10 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *70185570737580
+  version_requirements: *70328583624960
 - !ruby/object:Gem::Dependency
   name: shoulda
-  requirement: &70185570737100 !ruby/object:Gem::Requirement
+  requirement: &70328583624380 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -120,10 +131,10 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *70185570737100
+  version_requirements: *70328583624380
 - !ruby/object:Gem::Dependency
   name: mocha
-  requirement: &70185570736600 !ruby/object:Gem::Requirement
+  requirement: &70328583623720 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -131,13 +142,12 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *70185570736600
+  version_requirements: *70328583623720
 description: This gem implements algorithm for fuzzy matching scientific names developed
   by Tony Rees
 email: dmozzherin@eol.org
 executables: []
-extensions:
-- ext/damerau_levenshtein/extconf.rb
+extensions: []
 extra_rdoc_files:
 - LICENSE
 - README.rdoc
@@ -149,20 +159,16 @@ files:
 - README.rdoc
 - Rakefile
 - VERSION
-- ext/damerau_levenshtein/damerau_levenshtein.c
 - lib/taxamatch_rb.rb
 - lib/taxamatch_rb/atomizer.rb
 - lib/taxamatch_rb/authmatch.rb
-- lib/taxamatch_rb/damerau_levenshtein_mod.rb
 - lib/taxamatch_rb/normalizer.rb
 - lib/taxamatch_rb/phonetizer.rb
-- spec/damerau_levenshtein_mod_test.txt
 - spec/spec.opts
 - spec/spec_helper.rb
 - spec/taxamatch_rb_spec.rb
 - spec/taxamatch_test.txt
 - taxamatch_rb.gemspec
-- ext/damerau_levenshtein/extconf.rb
 homepage: http://github.com/GlobalNamesArchitecture/taxamatch_rb
 licenses: []
 post_install_message: 
@@ -177,7 +183,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: -4043039729385044436
+      hash: 1595435064862339145
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements:

data/ext/damerau_levenshtein/damerau_levenshtein.c

@@ -1,112 +0,0 @@
-#include "ruby.h"
-
-VALUE DamerauLevenshtein = Qnil;
-
-void Init_damerau_levenshtein();
-
-VALUE method_distance_utf(VALUE self, VALUE _s, VALUE _t, VALUE _block_size, VALUE _max_distance);
-
-void Init_damerau_levenshtein() {
-	DamerauLevenshtein = rb_define_module("DamerauLevenshtein");
-	rb_define_method(DamerauLevenshtein, "distance_utf", method_distance_utf, 4);
-}
-
-VALUE method_distance_utf(VALUE self, VALUE _s, VALUE _t, VALUE _block_size, VALUE _max_distance){
-  int i, i1, j, j1, k, half_tl, cost, *d, distance, del, ins, subs, transp, block;
-  int sl, tl, half_sl;
-  int stop_execution = 0;
-  int min = 0;
-  int current_distance = 0;
-
-  int block_size = NUM2INT(_block_size);
-  int max_distance = NUM2INT(_max_distance);
-
-  VALUE *sv = RARRAY_PTR(_s);
-  VALUE *tv = RARRAY_PTR(_t);
-
-  sl = (int) RARRAY_LEN(_s);
-  tl = (int) RARRAY_LEN(_t);
-
-  if (sl == 0) return INT2NUM(tl);
-  if (tl == 0) return INT2NUM(sl);
-  //case of lengths 1 must present or it will break further in the code
-  if (sl == 1 && tl == 1 && sv[0] != tv[0]) return INT2NUM(1);
-
-  int s[sl];
-  int t[tl];
-
-  for (i=0; i < sl; i++) s[i] = NUM2INT(sv[i]);
-  for (i=0; i < tl; i++) t[i] = NUM2INT(tv[i]);
-
-  sl++;
-  tl++;
-
-  //one-dimentional representation of 2 dimentional array len(s)+1 * len(t)+1
-  d = malloc((sizeof(int))*(sl)*(tl));
-  //populate 'vertical' row starting from the 2nd position (first one is filled already)
-  for(i = 0; i < tl; i++){
-    d[i*sl] = i;
-  }
-
-  //fill up array with scores
-  for(i = 1; i<sl; i++){
-    d[i] = i;
-    if (stop_execution == 1) break;
-    current_distance = 10000;
-    for(j = 1; j<tl; j++){
-
-      cost = 1;
-      if(s[i-1] == t[j-1]) cost = 0;
-
-      half_sl = (sl - 1)/2;
-      half_tl = (tl - 1)/2;
-
-      block = block_size < half_sl ? block_size : half_sl;
-      block = block < half_tl ? block : half_tl;
-
-      while (block >= 1){
-        int swap1 = 1;
-        int swap2 = 1;
-        i1 = i - (block * 2);
-        j1 = j - (block * 2);
-        for (k = i1; k < i1 + block; k++) {
-          if (s[k] != t[k + block]){
-            swap1 = 0;
-            break;
-          }
-        }
-        for (k = j1; k < j1 + block; k++) {
-          if (t[k] != s[k + block]){
-            swap2 = 0;
-            break;
-          }
-        }
-
-        del = d[j*sl + i - 1] + 1;
-        ins = d[(j-1)*sl + i] + 1;
-        min = del;
-        if (ins < min) min = ins;
-        //if (i == 2 && j==2) return INT2NUM(swap2+5);
-        if (i >= block && j >= block && swap1 == 1 && swap2 == 1){
-          transp = d[(j - block * 2) * sl + i - block * 2] + cost + block -1;
-          if (transp < min) min = transp;
-          block = 0;
-        } else if (block == 1) {
-          subs = d[(j-1)*sl + i - 1] + cost;
-          if (subs < min) min = subs;
-        }
-        block--;
-      }
-      d[j*sl+i]=min;
-      if (current_distance > d[j*sl+i]) current_distance = d[j*sl+i];
-    }
-    if (current_distance > max_distance) {
-      stop_execution = 1;
-    }
-  }
-  distance=d[sl * tl - 1];
-  if (stop_execution == 1) distance = current_distance;
-
-  free(d);
-  return INT2NUM(distance);
-}

data/ext/damerau_levenshtein/extconf.rb

@@ -1,11 +0,0 @@
-# Loads mkmf which is used to make makefiles for Ruby extensions
-require 'mkmf'
-
-# Give it a name
-extension_name = 'damerau_levenshtein'
-
-# The destination
-dir_config(extension_name)
-
-# Do the work
-create_makefile(extension_name)

data/lib/taxamatch_rb/damerau_levenshtein_mod.rb

@@ -1,42 +0,0 @@
-# encoding: UTF-8
-
-require 'damerau_levenshtein'
-
-module Taxamatch
-
-  class DamerauLevenshteinMod
-    include DamerauLevenshtein
-
-    def distance(str1, str2, block_size=2, max_distance=10)
-      distance_utf(str1.unpack("U*"), str2.unpack("U*"), block_size, max_distance)
-    end
-  end
-
-end
-
-if __FILE__ == $0
-
-  a = Taxamatch::DamerauLevenshteinMod.new
-  s = 'Cedarinia scabra Sjöstedt 1921'.unpack('U*')
-  t = 'Cedarinia scabra Söjstedt 1921'.unpack('U*')
-
-  #puts s.join(",")
-  #puts t.join(",")
-
-  start = Time.now
-  (1..100000).each do
-   a.distance('Cedarinia scabra Sjöstedt 1921', 'Cedarinia scabra Söjstedt 1921',1,10)
-  end
-  puts "with unpack time: " + (Time.now - start).to_s + ' sec'
-
-  start = Time.now
-  (1..100000).each do
-   a.distance_utf(s, t, 1, 10)
-  end
-  puts 'utf time: ' + (Time.now - start).to_s + ' sec'
-
-  #puts a.distance('Cedarinia scabra Sjöstedt 1921','Cedarinia scabra Söjstedt 1921')
-  #puts a.distance_utf(s, t, 2, 10)
-  #puts a.distance('tar','atp',1,10);
-  puts a.distance('sub', 'usb', 1, 10);
-end

data/spec/damerau_levenshtein_mod_test.txt

@@ -1,63 +0,0 @@
-######################
-# Tests for modified Damerau Levenshtein Distance algorithm (UTF-8 compatible)
-#
-# * B. Boehmer, T. Rees, Modified Damerau-Levenshtein Distance, Boehmer & Rees 2008
-# * F.J. Damerau. A technique for computer detection and correction of spelling errors, Communications of the ACM, 1964
-#
-# Fields:
-# String1|String2|maximum distance|transposition block size|expected distance
-#  - String1, String2
-#     compared strings
-#  - maximum distance
-#     stops execution of the algorithm when calculated distance exceeds the maximum distance number
-#  - transosition block size
-#     determines how many characters can be transposed. Block size 1 returns score according to Damerau-Levenshtein algorithm
-#  - expected distance
-#     resulting distance that has to be achieved by the algorithm
-#  Note: algorithm does not try to normalize or interpret strings in any way.
-######################
-
-#it whould recognize the exact match
-Pomatomus|Pomatomus|10|1|0
-
-#it should not try to normalize incoming strings
- Pomatomus|Pomatomus|10|1|1
-Pomatomus|pomatomus|10|1|1
-
-#it should calculate special cases
-Pomatomus||10|1|9
-|Pomatomus|10|1|9
-P|p|10|1|1
-#TODO: one letter vs longer string generates a big negative number
-#L|Linneaus|10|1|7
-
-
-#it should calculate Damerau Levenshtein distance with 1 character transpositions, insertions, deletions, substitutions (block size 1)
-Pomatomus|Pomatomux|10|1|1
-Pmatomus|Pomatomus|10|1|1
-Pomatomus|Pmatomus|10|1|1
-Rpmatomus|Pomatomus|10|1|2
-Pommtomus|Pomatomus|10|1|1
-Potamomus|Pomatomus|10|1|2
-Cedarinia scabra Sjöstedt 1921|Cedarinia scabra Sjostedt 1921|10|1|1
-Pomatomus|oPmatomus|10|1|1
-Pomatomus|Pomatomsu|10|1|1
-Pomtaomus|Pomatomus|10|1|1
-Pomatoums|Pomatomus|10|1|1
-Potamomus|Pomatomus|10|1|2
-Cedarinia scabra Sjöstedt 1921|Cedarinia scabra Söjstedt 1921|10|2|1
-
-#it should calculate Modified Damerau Levenshtein distance with 2 or more characters transposition (block size > 2)
-serrulatus|serratulus|10|2|2
-Pomatomus|Poomumats|10|3|3
-vesiculosus|vecusilosus|10|1|4
-vesiculosus|vecusilosus|10|2|2
-trimerophyton|mertriophyton|10|1|6
-trimerophyton|mertriophyton|10|3|3
-
-#it should stop trying if distance exceeds maximum allowed distance
-Pxxxxomus|Pomatomus|10|1|4
-Pxxxxomus|Pomatomus|2|1|3
-
-#
-PUNCTATA|PUNCTATA|10|1|0