hotwater 0.1.0

data/.gitignore

@@ -0,0 +1,19 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp
+.ruby-version
+vendor/

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in hotwater.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,45 @@
+Copyright (c) 2013 Colin Surprenant <colin.surprenant@gmail.com>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+--------
+
+C code from the https://github.com/sunlightlabs/jellyfish project
+
+Copyright (c) 2010, Sunlight Labs
+
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification,
+are permitted provided that the following conditions are met:
+
+    * Redistributions of source code must retain the above copyright notice, 
+      this list of conditions and the following disclaimer.
+    * Redistributions in binary form must reproduce the above copyright notice, 
+      this list of conditions and the following disclaimer in the documentation 
+      and/or other materials provided with the distribution.
+    * Neither the name of Sunlight Labs nor the names of its contributors may be
+      used to endorse or promote products derived from this software without 
+      specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

data/README.md

@@ -0,0 +1,70 @@
+# Hotwater v0.1.0
+
+Ruby & JRuby gem with fast **string edit distance** C implementations using FFI bindings.
+
+### Algorithms
+
+- Levenshtein & Damerau Levenshtein
+- Jaro & Jaro Winkler
+- N-Gram
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'hotwater'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install hotwater
+
+## Usage
+
+```ruby
+Hotwater.levenshtein_distance("abc", "acb") # => 2
+Hotwater.damerau_levenshtein_distance("abc", "acb") # => 1
+
+# do normalization based on the string sizes
+Hotwater.normalized_levenshtein_distance("abc", "acb").round(4) # => 0.3333
+Hotwater.normalized_damerau_levenshtein_distance("abc", "acb").round(4) # => 0.6667
+
+Hotwater.jaro_distance("martha", "marhta").round(4) # => 0.9444
+Hotwater.jaro_winkler_distance("martha", "marhta").round(4) # => 0.9611
+
+# default is bigram
+Hotwater.ngram_distance("natural", "contrary").round(4) # => 0.25
+
+# specify trigram
+Hotwater.ngram_distance("natural", "contrary", 3).round(4) # => 0.2083
+```
+
+## Developement
+
+1. Fort it
+2. Install gems `$ bundle install`
+3. Compile lib `$ rake compile`
+4. Run specs `$ rake spec`
+5. Clean compiler generated files `$ rake clean`
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request
+
+## Credits
+- Some C code from the https://github.com/sunlightlabs/jellyfish project
+- N-Gram ported from Apache Lucene 4.0.0 NGramDistance.java 
+
+## Author
+Colin Surprenant, [@colinsurprenant](http://twitter.com/colinsurprenant), [http://github.com/colinsurprenant](http://github.com/colinsurprenant), colin.surprenant@gmail.com
+
+## License
+Hotwater is distributed under the Apache License, Version 2.0. 
+

data/Rakefile

@@ -0,0 +1,25 @@
+require 'bundler/setup'
+require 'rake'
+require 'rake/clean'
+require 'bundler/gem_tasks'
+require 'rspec/core/rake_task'
+require 'ffi-compiler/compile_task'
+
+task :default => :spec
+
+desc "run specs"
+task :spec do
+  RSpec::Core::RakeTask.new
+end
+
+desc "compiler tasks"
+namespace "ffi-compiler" do
+  FFI::Compiler::CompileTask.new('ext/hotwater/hotwater') do |c|
+  end
+end
+task :compile => ["ffi-compiler:default"]
+
+CLEAN.include('ext/**/*{.o,.log,.so,.bundle}')
+CLEAN.include('lib/**/*{.o,.log,.so,.bundle}')
+CLEAN.include('ext/**/Makefile')
+

data/ext/hotwater/Rakefile

@@ -0,0 +1,5 @@
+require 'ffi-compiler/compile_task'
+
+FFI::Compiler::CompileTask.new('hotwater') do |c|
+  # nothing yet bro
+end

data/ext/hotwater/damerau_levenshtein.c

@@ -0,0 +1,68 @@
+/* from the https://github.com/sunlightlabs/jellyfish project */
+
+#include "hotwater.h"
+#include <string.h>
+
+int damerau_levenshtein_distance(const char *s1, const char *s2)
+{
+    size_t s1_len = s1 == NULL ? 0 : strlen(s1);
+    size_t s2_len = s2 == NULL ? 0 : strlen(s2);
+    size_t rows = s1_len + 1;
+    size_t cols = s2_len + 1;
+
+    size_t i, j;
+    size_t d1, d2, d3, d_now;;
+    unsigned short cost;
+
+    if (s1_len == 0) {
+        if (s2_len == 0) {
+            return 0;
+        }
+        else
+        {
+            return s2_len;
+        }
+    }
+
+    size_t *dist = malloc(rows * cols * sizeof(size_t));
+    if (!dist) {
+        return -1;
+    }
+
+    for (i = 0; i < rows; i++) {
+        dist[i * cols] = i;
+    }
+
+    for (j = 0; j < cols; j++) {
+        dist[j] = j;
+    }
+
+    for (i = 1; i < rows; i++) {
+        for (j = 1; j < cols; j++) {
+            if (s1[i - 1] == s2[j - 1]) {
+                cost = 0;
+            } else {
+                cost = 1;
+            }
+
+            d1 = dist[((i - 1) * cols) + j] + 1;
+            d2 = dist[(i * cols) + (j - 1)] + 1;
+            d3 = dist[((i - 1) * cols) + (j - 1)] + cost;
+
+            d_now = MIN(d1, MIN(d2, d3));
+
+            if (i > 2 && j > 2 && s1[i - 1] == s2[j - 2] &&
+                s1[i - 2] == s2[j - 1]) {
+                d1 = dist[((i - 2) * cols) + (j - 2)] + cost;
+                d_now = MIN(d_now, d1);
+            }
+
+            dist[(i * cols) + j] = d_now;
+        }
+    }
+
+    d_now = dist[(cols * rows) - 1];
+    free(dist);
+
+    return d_now;
+}

data/ext/hotwater/hotwater.h

@@ -0,0 +1,23 @@
+#ifndef _HOTWATER_H_
+#define _HOTWATER_H_
+
+#include <stdbool.h>
+#include <stdlib.h>
+
+#ifndef MIN
+  #define MIN(a, b) (((a) < (b)) ? (a) : (b))
+#endif
+
+#ifndef MAX
+  #define MAX(a, b) (((a) > (b)) ? (a) : (b))
+#endif
+
+double jaro_winkler_distance(const char *str1, const char *str2, bool long_tolerance);
+double jaro_distance(const char *str1, const char *str2);
+
+int levenshtein_distance(const char *str1, const char *str2);
+int damerau_levenshtein_distance(const char *str1, const char *str2);
+
+double ngram_distance(const char *source, const char *target, int n);
+
+#endif /* _HOTWATER_H_ */

data/ext/hotwater/jaro.c

@@ -0,0 +1,144 @@
+/* from the https://github.com/sunlightlabs/jellyfish project */
+
+/* 
+Colin Surprenant, Feb 2013
+- modified error return to -1 and small cosmetic cleanups
+*/
+
+
+#include <ctype.h>
+#include <string.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include "hotwater.h"
+
+#define NOTNUM(c)   ((c>57) || (c<48))
+#define INRANGE(c)  ((c>0)  && (c<91))
+
+/* borrowed heavily from strcmp95.c
+ *    http://www.census.gov/geo/msb/stand/strcmp.c
+ */
+double _jaro_winkler(const char *ying, const char *yang, bool long_tolerance, bool winklerize)
+{
+    /* Arguments:
+
+       ying
+       yang
+         pointers to the 2 strings to be compared.
+
+       long_tolerance
+         Increase the probability of a match when the number of matched
+         characters is large.  This option allows for a little more
+         tolerance when the strings are large.  It is not an appropriate
+         test when comparing fixed length fields such as phone and
+         social security numbers.
+    */
+    char *ying_flag = 0, *yang_flag = 0;
+
+    double weight;
+
+    long ying_length, yang_length, min_len;
+    long search_range;
+    long lowlim, hilim;
+    long trans_count, common_chars;
+
+    int i, j, k;
+
+    // ensure that neither string is blank
+    ying_length = strlen(ying);
+    yang_length = strlen(yang);
+    if (!ying_length || !yang_length) return 0;
+
+    search_range = min_len = (ying_length > yang_length) ? ying_length : yang_length;
+
+    // Blank out the flags
+    ying_flag = alloca(ying_length + 1);
+    if (!ying_flag) return -1.0;
+
+    yang_flag = alloca(yang_length + 1);
+    if (!yang_flag) return -1.0;
+
+    memset(ying_flag, 0, ying_length + 1);
+    memset(yang_flag, 0, yang_length + 1);
+
+    search_range = (search_range/2) - 1;
+    if (search_range < 0) search_range = 0;
+
+
+    // Looking only within the search range, count and flag the matched pairs.
+    common_chars = 0;
+    for (i = 0; i < ying_length; i++) {
+        lowlim = (i >= search_range) ? i - search_range : 0;
+        hilim = (i + search_range <= yang_length-1) ? (i + search_range) : yang_length-1;
+        for (j = lowlim; j <= hilim; j++)  {
+            if (!yang_flag[j] && yang[j] == ying[i]) {
+                yang_flag[j] = 1;
+                ying_flag[i] = 1;
+                common_chars++;
+                break;
+            }
+        }
+    }
+
+    // If no characters in common - return
+    if (!common_chars) return 0;
+
+    // Count the number of transpositions
+    k = trans_count = 0;
+    for (i = 0; i < ying_length; i++) {
+        if (ying_flag[i]) {
+            for (j = k; j < yang_length; j++) {
+                if (yang_flag[j]) {
+                    k = j + 1;
+                    break;
+                }
+            }
+            if (ying[i] != yang[j]) {
+                trans_count++;
+            }
+        }
+    }
+    trans_count /= 2;
+
+    // adjust for similarities in nonmatched characters
+
+    // Main weight computation.
+    weight= common_chars / ((double) ying_length) + common_chars / ((double) yang_length)
+        + ((double) (common_chars - trans_count)) / ((double) common_chars);
+    weight /= 3.0;
+
+    // Continue to boost the weight if the strings are similar
+    if (winklerize && weight > 0.7) {
+
+        // Adjust for having up to the first 4 characters in common
+        j = (min_len >= 4) ? 4 : min_len;
+        for (i=0; ((i<j) && (ying[i] == yang[i]) && (NOTNUM(ying[i]))); i++);
+        if (i) {
+            weight += i * 0.1 * (1.0 - weight);
+        }
+
+        /* Optionally adjust for long strings. */
+        /* After agreeing beginning chars, at least two more must agree and
+           the agreeing characters must be > .5 of remaining characters.
+        */
+        if ((long_tolerance) && (min_len>4) && (common_chars>i+1) && (2*common_chars>=min_len+i)) {
+            if (NOTNUM(ying[0])) {
+                weight += (double) (1.0-weight) *
+                    ((double) (common_chars-i-1) / ((double) (ying_length+yang_length-i*2+2)));
+            }
+        }
+    }
+
+    return weight;
+}
+
+
+double jaro_winkler_distance(const char *ying, const char *yang, bool long_tolerance)
+{
+    return _jaro_winkler(ying, yang, long_tolerance, true);
+}
+
+double jaro_distance(const char *ying, const char *yang)
+{
+    return _jaro_winkler(ying, yang, false, false);
+}

data/ext/hotwater/levenshtein.c

@@ -0,0 +1,51 @@
+/* from the https://github.com/sunlightlabs/jellyfish project */
+
+#include "hotwater.h"
+#include <string.h>
+#include <stdlib.h>
+#include <stdio.h>
+
+int levenshtein_distance(const char *s1, const char *s2)
+{
+    size_t s1_len = strlen(s1);
+    size_t s2_len = strlen(s2);
+    size_t rows = s1_len + 1;
+    size_t cols = s2_len + 1;
+    size_t i, j;
+
+    unsigned result;
+    unsigned d1, d2, d3;
+    unsigned *dist = malloc(rows * cols * sizeof(unsigned));
+    if (!dist) {
+        return -1;
+    }
+
+    for (i = 0; i < rows; i++) {
+        dist[i * cols] = i;
+    }
+
+
+    for (j = 0; j < cols; j++) {
+        dist[j] = j;
+    }
+
+    for (j = 1; j < cols; j++) {
+        for (i = 1; i < rows; i++) {
+            if (s1[i - 1] == s2[j - 1]) {
+                dist[(i * cols) + j] = dist[((i - 1) * cols) + (j - 1)];
+            } else {
+                d1 = dist[((i - 1) * cols) + j] + 1;
+                d2 = dist[(i * cols) + (j - 1)] + 1;
+                d3 = dist[((i - 1) * cols) + (j - 1)] + 1;
+
+                dist[(i * cols) + j] = MIN(d1, MIN(d2, d3));
+            }
+        }
+    }
+
+    result = dist[(cols * rows) - 1];
+
+    free(dist);
+
+    return result;
+}

data/ext/hotwater/ngram.c

@@ -0,0 +1,173 @@
+/* 
+Colin Surprenant, Feb 2013
+- converted in C from org/apache/lucene/search/spell/NGramDistance.java v4.0.0
+- fixed segfault bug in substring n parameter, which did not surface in Java
+*/
+
+/* package org.apache.lucene.search.spell; */
+
+/**
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements.  See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License.  You may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+/**
+ * N-Gram version of edit distance based on paper by Grzegorz Kondrak, 
+ * "N-gram similarity and distance". Proceedings of the Twelfth International 
+ * Conference on String Processing and Information Retrieval (SPIRE 2005), pp. 115-126, 
+ * Buenos Aires, Argentina, November 2005. 
+ * http://www.cs.ualberta.ca/~kondrak/papers/spire05.pdf
+ * 
+ * This implementation uses the position-based optimization to compute partial
+ * matches of n-gram sub-strings and adds a null-character prefix of size n-1 
+ * so that the first character is contained in the same number of n-grams as 
+ * a middle character.  Null-character prefix matches are discounted so that 
+ * strings with no matching characters will return a distance of 0.
+ * 
+ */
+
+#include "hotwater.h"
+#include <string.h>
+#include <stdlib.h>
+#include <stdio.h>
+
+char* substring(const char* s, int offset, int n) {
+  if (s == 0 || strlen(s) == 0 || strlen(s) < offset || strlen(s) < (offset + n)) {
+    return 0;    
+  }
+  return strndup(s + offset, n);
+}
+
+double ngram_distance (const char *source, const char *target, int n) {
+  int sl = strlen(source);
+  int tl = strlen(target);
+    
+  if (sl == 0 || tl == 0) {
+    if (sl == tl) {
+      return 1;
+    }
+    else {
+      return 0;
+    }
+  }
+
+  int cost = 0;
+  if (sl < n || tl < n) {
+    int ni = MIN(sl, tl);
+    for (int i = 0; i < ni; i++) {
+      if (source[i] == target[i]) {
+        cost++;
+      }
+    }
+    return (double)cost / (double)MAX(sl, tl);
+  }
+
+  int sa_len = sl + n - 1;
+  char* sa = calloc(sa_len + 1, sizeof(char));
+  if (!sa) {
+    return -1;
+  }
+
+  double* p; // 'previous' cost array, horizontally
+  double* d; // cost array, horizontally
+  double* _d; // placeholder to assist in swapping p and d
+  
+  // construct sa with prefix
+  for (int i = 0; i < sa_len; i++) {
+    if (i < n - 1) {
+      sa[i] = 0 ; //add prefix
+    }
+    else {
+      sa[i] = source[i - n + 1];
+    }
+  }
+
+  int p_d_len = sl + 1;
+  p = calloc(p_d_len + 1, sizeof(double)); 
+  if (!p) {
+    free(sa);
+    return -1;
+  }
+  d = calloc(p_d_len + 1, sizeof(double)); 
+  if (!d) {
+    free(sa);
+    free(p);
+    return -1;
+  }
+  
+  // indexes into strings s and t
+  int i = 0; // iterates through source
+  int j = 0; // iterates through target
+  
+  char* t_j = calloc(n + 1, sizeof(char)); // jth n-gram of t
+  if (!t_j) {
+    free(sa);
+    free(p);
+    free(d);
+    return -1;
+  }
+  for (i = 0; i <= sl; i++) {
+    p[i] = i;
+  }
+  
+  for (j = 1; j <= tl; j++) {
+    // construct t_j n-gram 
+    if (j < n) {
+      for (int ti = 0; ti < n - j; ti++) {
+        t_j[ti] = 0; //add prefix
+      }
+      for (int ti = n - j; ti < n; ti++) {
+         t_j[ti] = target[ti - (n - j)];
+      }
+    }
+    else {
+      free(t_j);
+      t_j = substring(target, j - n, n);
+    }
+    d[0] = j;
+    for (i = 1; i <= sl; i++) {
+      cost = 0;
+      int tn = n;
+      // compare sa to t_j
+
+      for (int ni = 0; ni < n; ni++) {
+        if (sa[i - 1 + ni] != t_j[ni]) {
+          cost++;
+        }
+        else if (sa[i - 1 + ni] == 0) { //discount matches on prefix
+          tn--;
+        }
+      }
+      double ec = (double)cost / (double)tn;
+
+      // minimum of cell to the left+1, to the top+1, diagonally left and up +cost
+      d[i] = MIN(MIN(d[i - 1] + 1, p[i] + 1), p[i - 1] + ec);
+    }
+    // copy current distance counts to 'previous row' distance counts
+    _d = p;
+    p = d;
+    d = _d;
+  }
+  double p_sl = p[sl];
+
+  free(p);
+  free(d);
+  free(t_j);
+  free(sa);
+
+  // our last action in the above loop was to switch d and p, so p now
+  // actually has the most recent cost counts
+  return 1.0 - (p_sl / (double)MAX(tl, sl));
+}

data/hotwater.gemspec

@@ -0,0 +1,26 @@
+# -*- encoding: utf-8 -*-
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'hotwater/version'
+
+Gem::Specification.new do |gem|
+  gem.name          = "hotwater"
+  gem.version       = Hotwater::VERSION
+  gem.authors       = ["Colin Surprenant"]
+  gem.email         = ["colin.surprenant@gmail.com"]
+  gem.description   = "Ruby & JRuby gem with fast string edit distance C implementation using FFI bindings"
+  gem.summary       = "Fast string edit distance"
+  gem.homepage      = "http://github.com/colinsurprenant/hotwater"
+
+  gem.files         = `git ls-files`.split($/)
+  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.require_paths = ["lib"]
+  gem.extensions    = ["ext/hotwater/Rakefile"]
+
+  gem.add_dependency 'rake'
+  gem.add_dependency 'ffi'
+  gem.add_dependency 'ffi-compiler'
+  
+  gem.add_development_dependency 'rspec'
+end

data/lib/hotwater/damerau_levenshtein_ffi.rb

@@ -0,0 +1,26 @@
+require 'ffi'
+
+module Hotwater
+  
+  module C
+    # int damerau_levenshtein_distance(const char *str1, const char *str2)
+    attach_function :damerau_levenshtein_distance, [:string, :string], :int
+  end
+
+  def damerau_levenshtein_distance(s1, s2)
+    result = C::damerau_levenshtein_distance(s1, s2)
+    raise("memory allocation error") if result == -1
+    result
+  end
+
+  def normalized_damerau_levenshtein_distance(s1, s2)
+    result = C::damerau_levenshtein_distance(s1, s2)
+    raise("memory allocation error") if result == -1
+    return 0.0 if result == 0
+    max = [s1.size, s2.size].max
+    (max - result.to_f) / max
+  end
+
+  module_function :damerau_levenshtein_distance, :normalized_damerau_levenshtein_distance
+
+end

data/lib/hotwater/jaro_ffi.rb

@@ -0,0 +1,26 @@
+require 'ffi'
+
+module Hotwater
+  
+  module C
+    # double jaro_distance(const char *ying, const char *yang)
+    attach_function :jaro_distance, [:string, :string], :double
+
+    # double jaro_winkler_distance(const char *ying, const char *yang, bool long_tolerance)
+    attach_function :jaro_winkler_distance, [:string, :string, :bool], :double
+  end
+
+  def jaro_distance(s1, s2)
+    result = C::jaro_distance(s1, s2)
+    raise("memory allocation error") if result < 0.0
+    result
+  end
+
+  def jaro_winkler_distance(s1, s2, long_tolerance = false)
+    result = C::jaro_winkler_distance(s1, s2, long_tolerance)
+    raise("memory allocation error") if result < 0.0
+    result
+  end
+
+  module_function :jaro_distance, :jaro_winkler_distance
+end

data/lib/hotwater/levenshtein_ffi.rb

@@ -0,0 +1,25 @@
+require 'ffi'
+
+module Hotwater
+	
+  module C
+    # int levenshtein_distance(const char *str1, const char *str2);
+    attach_function :levenshtein_distance, [:string, :string], :int
+  end
+
+  def levenshtein_distance(s1, s2)
+    result = C::levenshtein_distance(s1, s2)
+    raise("memory allocation error") if result == -1
+    result
+  end
+
+  def normalized_levenshtein_distance(s1, s2)
+    result = C::levenshtein_distance(s1, s2)
+    raise("memory allocation error") if result == -1
+    return 0.0 if result == 0
+    max = [s1.size, s2.size].max
+    (max - result.to_f) / max
+  end
+
+  module_function :levenshtein_distance, :normalized_levenshtein_distance
+end

data/lib/hotwater/ngram_ffi.rb

@@ -0,0 +1,18 @@
+require 'ffi'
+
+module Hotwater
+  
+  module C
+    # double ngram_distance(const char str1*, const char str2*, int n, int mode);
+    attach_function :ngram_distance, [:string, :string, :int], :double
+  end
+
+  def ngram_distance(s1, s2, n = 2)
+    result = C::ngram_distance(s1, s2, n)
+    raise("memory allocation error") if result == -1
+    result
+  end
+
+  module_function :ngram_distance
+
+end

data/lib/hotwater/version.rb

@@ -0,0 +1,3 @@
+module Hotwater
+  VERSION = "0.1.0"
+end

data/lib/hotwater.rb

@@ -0,0 +1,16 @@
+require 'hotwater/version'
+require 'ffi'
+require 'ffi-compiler/loader'
+
+module Hotwater
+  module C
+    extend FFI::Library
+    ffi_lib FFI::Compiler::Loader.find('hotwater')
+  end
+end
+
+require 'hotwater/levenshtein_ffi'
+require 'hotwater/damerau_levenshtein_ffi'
+require 'hotwater/jaro_ffi'
+require 'hotwater/ngram_ffi'
+

data/spec/hotwater/damerau_levenshtein_ffi_spec.rb

@@ -0,0 +1,29 @@
+require 'rspec'
+require 'hotwater'
+
+describe Hotwater do
+
+  it "should compute damerau_levenshtein_distance" do
+    Hotwater.damerau_levenshtein_distance("", "").should == 0
+    Hotwater.damerau_levenshtein_distance("abc", "").should == 3
+    Hotwater.damerau_levenshtein_distance("bc", "abc").should == 1
+    Hotwater.damerau_levenshtein_distance("ca", "abc").should == 3
+    Hotwater.damerau_levenshtein_distance("abc", "acb").should == 1
+    Hotwater.damerau_levenshtein_distance("kitten", "sitting").should == 3
+    Hotwater.damerau_levenshtein_distance("Saturday", "Sunday").should == 3
+    Hotwater.damerau_levenshtein_distance("teusday", "tuesday").should == 1
+    Hotwater.damerau_levenshtein_distance("teusday", "thursday").should == 2
+  end
+
+  it "should compute normalized_damerau_levenshtein_distance" do
+    Hotwater.normalized_damerau_levenshtein_distance("", "").should == 0.0
+    Hotwater.normalized_damerau_levenshtein_distance("abc", "").should == 0.0
+    Hotwater.normalized_damerau_levenshtein_distance("bc", "abc").round(4).should == 0.6667
+    Hotwater.normalized_damerau_levenshtein_distance("ca", "abc").should == 0.0
+    Hotwater.normalized_damerau_levenshtein_distance("abc", "acb").round(4).should == 0.6667
+    Hotwater.normalized_damerau_levenshtein_distance("kitten", "sitting").round(4).should == 0.5714
+    Hotwater.normalized_damerau_levenshtein_distance("Saturday", "Sunday").round(4).should == 0.625
+    Hotwater.normalized_damerau_levenshtein_distance("teusday", "tuesday").round(4).should == 0.8571
+    Hotwater.normalized_damerau_levenshtein_distance("teusday", "thursday").round(4).should == 0.75
+  end
+end

data/spec/hotwater/jaro_ffi_spec.rb

@@ -0,0 +1,19 @@
+require 'rspec'
+require 'hotwater'
+
+describe Hotwater do
+
+  it "should compute jaro_distance" do
+    Hotwater.jaro_distance("", "").should == 0.0
+    Hotwater.jaro_distance("dixon", "dicksonx").round(4).should == 0.7667
+    Hotwater.jaro_distance("martha", "marhta").round(4).should == 0.9444
+    Hotwater.jaro_distance("dwayne", "duane").round(4).should == 0.8222
+  end
+
+  it "should compute jaro_winkler_distance" do
+    Hotwater.jaro_winkler_distance("", "").should == 0.0
+    Hotwater.jaro_winkler_distance("dixon", "dicksonx").round(4).should == 0.8133
+    Hotwater.jaro_winkler_distance("martha", "marhta").round(4).should == 0.9611
+    Hotwater.jaro_winkler_distance("dwayne", "duane").round(4).should == 0.84
+  end
+end

data/spec/hotwater/levenshtein_ffi_spec.rb

@@ -0,0 +1,29 @@
+require 'rspec'
+require 'hotwater'
+
+describe Hotwater do
+
+  it "should compute levenshtein_distance" do
+    Hotwater.levenshtein_distance("", "").should == 0
+    Hotwater.levenshtein_distance("abc", "").should == 3
+    Hotwater.levenshtein_distance("bc", "abc").should == 1
+    Hotwater.levenshtein_distance("ca", "abc").should == 3
+    Hotwater.levenshtein_distance("abc", "acb").should == 2
+    Hotwater.levenshtein_distance("kitten", "sitting").should == 3
+    Hotwater.levenshtein_distance("Saturday", "Sunday").should == 3
+    Hotwater.levenshtein_distance("teusday", "tuesday").should == 2
+    Hotwater.levenshtein_distance("teusday", "thursday").should == 2
+  end
+
+  it "should compute normalized_levenshtein_distance" do
+    Hotwater.normalized_levenshtein_distance("", "").should == 0.0
+    Hotwater.normalized_levenshtein_distance("abc", "").should == 0.0
+    Hotwater.normalized_levenshtein_distance("bc", "abc").round(4).should == 0.6667
+    Hotwater.normalized_levenshtein_distance("ca", "abc").should == 0.0
+    Hotwater.normalized_levenshtein_distance("abc", "acb").round(4).should == 0.3333
+    Hotwater.normalized_levenshtein_distance("kitten", "sitting").round(4).should == 0.5714
+    Hotwater.normalized_levenshtein_distance("Saturday", "Sunday").round(4).should == 0.625
+    Hotwater.normalized_levenshtein_distance("teusday", "tuesday").round(4).should == 0.7143
+    Hotwater.normalized_levenshtein_distance("teusday", "thursday").round(4).should == 0.75
+  end
+end

data/spec/hotwater/ngram_ffi_spec.rb

@@ -0,0 +1,75 @@
+require 'rspec'
+require 'hotwater'
+
+describe Hotwater do
+
+  it "should compute unigram distance" do
+    Hotwater.ngram_distance("", "al", 1).round(4).should == 0.0
+    Hotwater.ngram_distance("al", "al", 1).round(4).should == 1.0
+    Hotwater.ngram_distance("a", "a", 1).round(4).should == 1.0
+    Hotwater.ngram_distance("b", "a", 1).round(4).should == 0.0
+    Hotwater.ngram_distance("martha", "marhta", 1).round(4).should == 0.6667
+    Hotwater.ngram_distance("jones", "johnson", 1).round(4).should == 0.4286
+    Hotwater.ngram_distance("natural", "contrary", 1).round(4).should == 0.25
+    Hotwater.ngram_distance("abcvwxyz", "cabvwxyz", 1).round(4).should == 0.75
+    Hotwater.ngram_distance("dwayne", "duane", 1).round(4).should == 0.6667
+    Hotwater.ngram_distance("dixon", "dicksonx", 1).round(4).should == 0.5
+    Hotwater.ngram_distance("six", "ten", 1).round(4).should == 0.0
+    Hotwater.ngram_distance("zac ephron", "zac efron", 1).round(4).should == Hotwater.ngram_distance("zac ephron", "kai ephron", 1).round(4)
+    Hotwater.ngram_distance("brittney spears", "britney spears", 1).round(4).should > Hotwater.ngram_distance("brittney spears", "brittney startzman", 1)
+    Hotwater.ngram_distance("12345678", "12890678", 1).round(4).should == Hotwater.ngram_distance("12345678", "72385698", 1)
+  end
+
+  it "should compute bigram distance" do
+    Hotwater.ngram_distance("", "al", 2).round(4).should == 0.0
+    Hotwater.ngram_distance("al", "al", 2).round(4).should == 1.0
+    Hotwater.ngram_distance("a", "a", 2).round(4).should == 1.0
+    Hotwater.ngram_distance("b", "a", 2).round(4).should == 0.0
+    Hotwater.ngram_distance("a", "aa", 2).round(4).should == 0.5
+    Hotwater.ngram_distance("martha", "marhta", 2).round(4).should == 0.6667
+    Hotwater.ngram_distance("jones", "johnson", 2).round(4).should == 0.4286
+    Hotwater.ngram_distance("natural", "contrary", 2).round(4).should == 0.25
+    Hotwater.ngram_distance("abcvwxyz", "cabvwxyz", 2).round(4).should == 0.625
+    Hotwater.ngram_distance("dwayne", "duane", 2).round(4).should == 0.5833
+    Hotwater.ngram_distance("dixon", "dicksonx", 2).round(4).should == 0.5
+    Hotwater.ngram_distance("six", "ten", 2).round(4).should == 0.0
+    Hotwater.ngram_distance("zac ephron", "zac efron", 2).round(4).should > Hotwater.ngram_distance("zac ephron", "kai ephron", 2).round(4)
+    Hotwater.ngram_distance("brittney spears", "britney spears", 2).round(4).should > Hotwater.ngram_distance("brittney spears", "brittney startzman", 2)
+    Hotwater.ngram_distance("0012345678", "0012890678", 2).round(4).should == Hotwater.ngram_distance("0012345678", "0072385698", 2)
+  end
+
+  it "should compute bigram distance by default" do
+    Hotwater.ngram_distance("", "al").round(4).should == 0.0
+    Hotwater.ngram_distance("al", "al").round(4).should == 1.0
+    Hotwater.ngram_distance("a", "a").round(4).should == 1.0
+    Hotwater.ngram_distance("b", "a").round(4).should == 0.0
+    Hotwater.ngram_distance("a", "aa").round(4).should == 0.5
+    Hotwater.ngram_distance("martha", "marhta").round(4).should == 0.6667
+    Hotwater.ngram_distance("jones", "johnson").round(4).should == 0.4286
+    Hotwater.ngram_distance("natural", "contrary").round(4).should == 0.25
+    Hotwater.ngram_distance("abcvwxyz", "cabvwxyz").round(4).should == 0.625
+    Hotwater.ngram_distance("dwayne", "duane").round(4).should == 0.5833
+    Hotwater.ngram_distance("dixon", "dicksonx").round(4).should == 0.5
+    Hotwater.ngram_distance("six", "ten").round(4).should == 0.0
+    Hotwater.ngram_distance("zac ephron", "zac efron").round(4).should > Hotwater.ngram_distance("zac ephron", "kai ephron").round(4)
+    Hotwater.ngram_distance("brittney spears", "britney spears").round(4).should > Hotwater.ngram_distance("brittney spears", "brittney startzman")
+    Hotwater.ngram_distance("0012345678", "0012890678").round(4).should == Hotwater.ngram_distance("0012345678", "0072385698")
+  end
+
+  it "should compute trigram distance" do
+    Hotwater.ngram_distance("", "al", 3).round(4).should == 0.0
+    Hotwater.ngram_distance("al", "al", 3).round(4).should == 1.0
+    Hotwater.ngram_distance("a", "a", 3).round(4).should == 1.0
+    Hotwater.ngram_distance("b", "a", 3).round(4).should == 0.0
+    Hotwater.ngram_distance("martha", "marhta", 3).round(4).should == 0.7222
+    Hotwater.ngram_distance("jones", "johnson", 3).round(4).should == 0.4762
+    Hotwater.ngram_distance("natural", "contrary", 3).round(4).should == 0.2083
+    Hotwater.ngram_distance("abcvwxyz", "cabvwxyz", 3).round(4).should == 0.5625
+    Hotwater.ngram_distance("dwayne", "duane", 3).round(4).should == 0.5278
+    Hotwater.ngram_distance("dixon", "dicksonx", 3).round(4).should == 0.4583
+    Hotwater.ngram_distance("six", "ten", 3).round(4).should == 0.0
+    Hotwater.ngram_distance("zac ephron", "zac efron", 3).round(4).should > Hotwater.ngram_distance("zac ephron", "kai ephron", 3).round(4)
+    Hotwater.ngram_distance("brittney spears", "britney spears", 3).round(4).should > Hotwater.ngram_distance("brittney spears", "brittney startzman", 3)
+    Hotwater.ngram_distance("0012345678", "0012890678", 3).round(4).should < Hotwater.ngram_distance("0012345678", "0072385698", 3)
+  end
+end

metadata

@@ -0,0 +1,143 @@
+--- !ruby/object:Gem::Specification
+name: hotwater
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+  prerelease: 
+platform: ruby
+authors:
+- Colin Surprenant
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-02-25 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: ffi
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: ffi-compiler
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Ruby & JRuby gem with fast string edit distance C implementation using
+  FFI bindings
+email:
+- colin.surprenant@gmail.com
+executables: []
+extensions:
+- ext/hotwater/Rakefile
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- ext/hotwater/Rakefile
+- ext/hotwater/damerau_levenshtein.c
+- ext/hotwater/hotwater.h
+- ext/hotwater/jaro.c
+- ext/hotwater/levenshtein.c
+- ext/hotwater/ngram.c
+- hotwater.gemspec
+- lib/hotwater.rb
+- lib/hotwater/damerau_levenshtein_ffi.rb
+- lib/hotwater/jaro_ffi.rb
+- lib/hotwater/levenshtein_ffi.rb
+- lib/hotwater/ngram_ffi.rb
+- lib/hotwater/version.rb
+- spec/hotwater/damerau_levenshtein_ffi_spec.rb
+- spec/hotwater/jaro_ffi_spec.rb
+- spec/hotwater/levenshtein_ffi_spec.rb
+- spec/hotwater/ngram_ffi_spec.rb
+homepage: http://github.com/colinsurprenant/hotwater
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+      segments:
+      - 0
+      hash: -289401610859280349
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+      segments:
+      - 0
+      hash: -289401610859280349
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.23
+signing_key: 
+specification_version: 3
+summary: Fast string edit distance
+test_files:
+- spec/hotwater/damerau_levenshtein_ffi_spec.rb
+- spec/hotwater/jaro_ffi_spec.rb
+- spec/hotwater/levenshtein_ffi_spec.rb
+- spec/hotwater/ngram_ffi_spec.rb