levenshtein 0.1.0

data/LICENSE

@@ -0,0 +1,15 @@
+# Copyright Erik Veenstra <levenshtein@erikveen.dds.nl>
+# 
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License,
+# version 2, as published by the Free Software Foundation.
+# 
+# This program is distributed in the hope that it will be
+# useful, but WITHOUT ANY WARRANTY; without even the implied
+# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+# PURPOSE. See the GNU General Public License for more details.
+# 
+# You should have received a copy of the GNU General Public
+# License along with this program; if not, write to the Free
+# Software Foundation, Inc., 59 Temple Place, Suite 330,
+# Boston, MA 02111-1307 USA.

data/README

@@ -0,0 +1,8 @@
+# The Levenshtein distance is a metric for measuring the amount of difference
+# between two sequences (i.e., the so called edit distance). The Levenshtein
+# distance between two strings is given by the minimum number of operations
+# needed to transform one string into the other, where an operation is an
+# insertion, deletion, or substitution of a single character.
+#
+# More information about the Levenshtein distance algorithm:
+# http://en.wikipedia.org/wiki/Levenshtein_distance .

data/VERSION

@@ -0,0 +1 @@
+0.1.0

data/ext/levenshtein/extconf.rb

@@ -0,0 +1,5 @@
+require "mkmf"
+
+dir_config("levenshtein")
+
+create_makefile("levenshtein/levenshtein_c")

data/ext/levenshtein/levenshtein_c.c

@@ -0,0 +1,122 @@
+#include "ruby.h"
+
+static VALUE levenshtein_distance_part2(VALUE self, VALUE rb_s1, VALUE rb_s2, VALUE rb_threshold) {
+  VALUE	rb_s3;
+  int	threshold;
+  int	l1, l2, l3;
+  char	*s1, *s2, *s3;
+  int	*prev_row, *curr_row;
+  int	col, row;
+  int	curr_row_min, result;
+
+  /* Convert Ruby's s1 to C's s1. */
+
+  rb_s1	= StringValue(rb_s1);
+  s1	= RSTRING(rb_s1)->ptr;
+  l1	= RSTRING(rb_s1)->len;
+
+  /* Convert Ruby's s2 to C's s2. */
+
+  rb_s2	= StringValue(rb_s2);
+  s2	= RSTRING(rb_s2)->ptr;
+  l2	= RSTRING(rb_s2)->len;
+
+  /* Convert Ruby's threshold to C's threshold. */
+
+  if (!NIL_P(rb_threshold)) {
+    threshold	= FIX2INT(rb_threshold);
+  } else {
+    threshold	= -1;
+  }
+
+  /* The Levenshtein Algorithm itself. */
+
+  /*       s1=              */
+  /*       ERIK             */
+  /*                        */
+  /*      01234             */
+  /* s2=V 11234             */
+  /*    E 21234             */
+  /*    E 32234             */
+  /*    N 43334 <- prev_row */
+  /*    S 54444 <- curr_row */
+  /*    T 65555             */
+  /*    R 76566             */
+  /*    A 87667             */
+ 
+  /* Allocate memory for both rows */
+
+  prev_row	= ALLOC_N(int, l1+1);
+  curr_row	= ALLOC_N(int, l1+1);
+
+  if ((prev_row == NULL) || (curr_row == NULL)) {
+    rb_raise(rb_eNoMemError, "out of memory");
+  }
+
+  /* Initialize the current row. */
+
+  for (col=0; col<=l1; col++) {
+    curr_row[col]	= col;
+  }
+
+  for (row=1; row<=l2; row++) {
+    /* Copy the current row to the previous row. */
+
+    memcpy(prev_row, curr_row, sizeof(int)*(l1+1));
+
+    /* Calculate the values of the current row. */
+
+    curr_row[0]		= row;
+    curr_row_min	= row;
+
+    for (col=1; col<=l1; col++) {
+      /* Equal (cost=0) or Substitution (cost=1). */
+
+      curr_row[col]	= prev_row[col-1] + ((s1[col-1] == s2[row-1]) ? 0 : 1);
+
+      /* Insertion if it's cheaper than substitution. */
+
+      if (prev_row[col]+1 < curr_row[col]) {
+        curr_row[col] = prev_row[col]+1;
+      }
+
+      /* Deletion if it's cheaper than substitution. */
+
+      if (curr_row[col-1]+1 < curr_row[col]) {
+        curr_row[col] = curr_row[col-1]+1;
+      }
+
+      /* Keep track of the minimum value on this row. */
+
+      if (curr_row[col] < curr_row_min) {
+        curr_row_min	= curr_row[col];
+      }
+    }
+
+    /* Return nil as soon as we exceed the threshold. */
+
+    if (threshold > -1 && curr_row_min >= threshold) {
+      free(prev_row);
+      free(curr_row);
+
+      return Qnil;
+    }
+  }
+
+  /* The result is the last value on the last row. */
+
+  result	= curr_row[l1];
+
+  free(prev_row);
+  free(curr_row);
+
+  /* Return the Ruby version of the result. */
+
+  return INT2FIX(result);
+}
+
+void Init_levenshtein_c() {
+  VALUE mLevenshtein	= rb_define_module("Levenshtein");
+
+  rb_define_singleton_method(mLevenshtein, "distance_part2_fast" , levenshtein_distance_part2, 3);
+}

data/lib/levenshtein.rb

@@ -0,0 +1,100 @@
+begin
+  require "levenshtein/levenshtein_c"
+rescue LoadError
+  begin
+    require "levenshtein_c"
+  rescue LoadError
+    $stderr.puts "WARNING: Couldn't find the fast C implementation of Levenshtein.distance_part2. Using the slow Ruby version instead."
+  end
+end
+
+# The Levenshtein distance is a metric for measuring the amount of difference
+# between two sequences (i.e., the so called edit distance). The Levenshtein
+# distance between two strings is given by the minimum number of operations
+# needed to transform one string into the other, where an operation is an
+# insertion, deletion, or substitution of a single character.
+#
+# More information about the Levenshtein distance algorithm:
+# http://en.wikipedia.org/wiki/Levenshtein_distance .
+
+module Levenshtein
+  # Returns the Levenshtein distance as a number bestween 0.0 and 1.0.
+  # It's basically the Levenshtein distance divided by the length of the longest string.
+
+  def self.normalized_distance(s1, s2, threshold=nil)
+    s1, s2	= s2, s1	if s1.length > s2.length	# s1 is the short one; s2 is the long one.
+
+    if s2.empty?
+      0.0	# Since s1.length < s2.length, s1 must be empty as well.
+    else
+      if threshold
+        if d = self.distance(s1, s2, (threshold*s2.length+1).to_i)
+          d.to_f/s2.length
+        else
+          nil
+        end
+      else
+        self.distance(s1, s2).to_f/s2.length
+      end
+    end
+  end
+
+  # Returns the Levenshtein distance between two byte strings.
+
+  def self.distance(s1, s2, threshold=nil)
+    s1, s2	= s2, s1	if s1.length > s2.length	# s1 is the short one; s2 is the long one.
+
+    # Handle some basic circumstances.
+
+    return 0.0		if s1 == s2
+    return s2.length	if s1.empty?
+    return nil		if threshold and (s2.length-s1.length) >= threshold
+    return nil		if threshold and (s1.scan(/./) - s2.scan(/./)).length >= threshold
+    return nil		if threshold and (s2.scan(/./) - s1.scan(/./)).length >= threshold
+
+    # Do the expensive calculation on a subset of the strings only, if possible.
+
+    b	= 0
+    e1	= s1.length-1
+    e2	= s2.length-1
+
+    while s1[b, 1] == s2[b, 1]
+      b += 1
+    end
+
+    while s1[e1, 1] == s2[e2, 1]
+      e1 -= 1
+      e2 -= 1
+    end
+
+    distance_part2(s1[b..e1], s2[b..e2], threshold)
+  end
+
+  def self.distance_part2(s1, s2, threshold)	# :nodoc:
+    if respond_to?(:distance_part2_fast)
+      distance_part2_fast(s1, s2, threshold)	# Implemented in C.
+    else
+      distance_part2_slow(s1, s2, threshold)	# Implemented in Ruby.
+    end
+  end
+
+  def self.distance_part2_slow(s1, s2, threshold)	# :nodoc:
+    row	= (0..s1.length).to_a
+
+    1.upto(s2.length) do |y|
+      prow	= row
+      row	= [y]
+
+      1.upto(s1.length) do |x|
+        row[x]	= [prow[x]+1, row[x-1]+1, prow[x-1]+(s1[x-1]==s2[y-1] ? 0 : 1)].min
+      end
+
+      # Stop analysing this string as soon as the best possible result for this string is bigger than the best result so far.
+      # (The minimum value in the next row will be equal to or greater than the minimum value in this row.)
+
+      return nil	if threshold and row.min >= threshold
+    end
+
+    row[-1]
+  end
+end

data/test/test.rb

@@ -0,0 +1,90 @@
+require "test/unit"
+require "levenshtein"
+
+class TestLevenshtein < Test::Unit::TestCase
+  def test_erik_veenstra
+    assert_equal(7, Levenshtein.distance("erik", "veenstra"))
+    assert_equal(7, Levenshtein.distance("veenstra", "erik"))
+
+    assert_in_delta(0.875, Levenshtein.normalized_distance("erik", "veenstra"), 0.01)
+    assert_in_delta(0.875, Levenshtein.normalized_distance("veenstra", "erik"), 0.01)
+  end
+
+  def test_empty_string
+    assert_equal(0, Levenshtein.distance("", ""))
+    assert_equal(3, Levenshtein.distance("", "foo"))
+    assert_equal(3, Levenshtein.distance("foo", ""))
+
+    assert_in_delta(0.0, Levenshtein.normalized_distance("", ""), 0.01)
+    assert_in_delta(1.0, Levenshtein.normalized_distance("", "foo"), 0.01)
+    assert_in_delta(1.0, Levenshtein.normalized_distance("foo", ""), 0.01)
+  end
+
+  def test_same_string
+    assert_equal(0, Levenshtein.distance("", ""))
+    assert_equal(0, Levenshtein.distance("foo", "foo"))
+
+    assert_in_delta(0.0, Levenshtein.normalized_distance("", ""), 0.01)
+    assert_in_delta(0.0, Levenshtein.normalized_distance("foo", "foo"), 0.01)
+  end
+
+  def test_threshold
+    assert_equal(3, Levenshtein.distance("foo", "foobar"))
+    assert_equal(nil, Levenshtein.distance("foo", "foobar", 2))
+
+    assert_in_delta(0.5, Levenshtein.normalized_distance("foo", "foobar"), 0.01)
+    assert_equal(nil, Levenshtein.normalized_distance("foo", "foobar", 0.30))
+  end
+
+  def test_same_head_and_or_tail
+    assert_equal(3, Levenshtein.distance("ab123cd", "abxyzcd"))
+    assert_equal(3, Levenshtein.distance("ab123", "abxyz"))
+    assert_equal(3, Levenshtein.distance("123cd", "xyzcd"))
+
+    assert_in_delta(0.42, Levenshtein.normalized_distance("ab123cd", "abxyzcd"), 0.01)
+    assert_in_delta(0.6, Levenshtein.normalized_distance("ab123", "abxyz"), 0.01)
+    assert_in_delta(0.6, Levenshtein.normalized_distance("123cd", "xyzcd"), 0.01)
+  end
+end
+
+class TestLevenshteinPart2Slow < Test::Unit::TestCase
+  def test_erik_veenstra
+    assert_equal(7, Levenshtein.distance_part2_slow("erik", "veenstra", nil))
+  end
+
+  def test_empty_string
+    assert_equal(0, Levenshtein.distance_part2_slow("", "", nil))
+    assert_equal(3, Levenshtein.distance_part2_slow("", "foo", nil))
+  end
+
+  def test_same_string
+    assert_equal(0, Levenshtein.distance_part2_slow("", "", nil))
+    assert_equal(0, Levenshtein.distance_part2_slow("foo", "foo", nil))
+  end
+
+  def test_threshold
+    assert_equal(3, Levenshtein.distance_part2_slow("foo", "foobar", nil))
+    assert_equal(nil, Levenshtein.distance_part2_slow("foo", "foobar", 2))
+  end
+end
+
+class TestLevenshteinPart2Fast < Test::Unit::TestCase
+  def test_erik_veenstra
+    assert_equal(7, Levenshtein.distance_part2_fast("erik", "veenstra", nil))
+  end
+
+  def test_empty_string
+    assert_equal(0, Levenshtein.distance_part2_fast("", "", nil))
+    assert_equal(3, Levenshtein.distance_part2_fast("", "foo", nil))
+  end
+
+  def test_same_string
+    assert_equal(0, Levenshtein.distance_part2_fast("", "", nil))
+    assert_equal(0, Levenshtein.distance_part2_fast("foo", "foo", nil))
+  end
+
+  def test_threshold
+    assert_equal(3, Levenshtein.distance_part2_fast("foo", "foobar", nil))
+    assert_equal(nil, Levenshtein.distance_part2_fast("foo", "foobar", 2))
+  end
+end

metadata

@@ -0,0 +1,65 @@
+--- !ruby/object:Gem::Specification 
+name: levenshtein
+version: !ruby/object:Gem::Version 
+  version: 0.1.0
+platform: ruby
+authors: 
+- Erik Veenstra
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2008-05-24 00:00:00 +02:00
+default_executable: 
+dependencies: []
+
+description: Calculates the Levenshtein distance between two byte strings.
+email: levenshtein@erikveen.dds.nl
+executables: []
+
+extensions: 
+- ext/levenshtein/extconf.rb
+extra_rdoc_files: []
+
+files: 
+- lib/levenshtein.rb
+- ext/levenshtein
+- ext/levenshtein/extconf.rb
+- ext/levenshtein/levenshtein_c.c
+- README
+- LICENSE
+- VERSION
+has_rdoc: true
+homepage: http://www.erikveen.dds.nl/levenshtein/index.html
+post_install_message: 
+rdoc_options: 
+- README
+- LICENSE
+- VERSION
+- --title
+- levenshtein (0.1.0)
+- --main
+- README
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+requirements: []
+
+rubyforge_project: levenshtein
+rubygems_version: 1.0.1
+signing_key: 
+specification_version: 2
+summary: Calculates the Levenshtein distance between two byte strings.
+test_files: 
+- test/test.rb