jaro_winkler 1.3.1 → 1.3.2.beta

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 77f956f70c8b182eae7e78f336c377bc1b4bfa8f
-  data.tar.gz: 17bead21ccd72a4d1c6137cf4f612b1d1c036e26
+  metadata.gz: 9a90ee9e013479d3c47c7cd632646921f87af4fc
+  data.tar.gz: 30d065fc0728d3fda6db84af71584f7a60b038cb
 SHA512:
-  metadata.gz: a4c0e7786621635addc80d7c0c18276ef720c1e9690c61f74b506cf5627368cb5862073e96447cef6739ab044d18aa159db492e3bdc182a483809bd273979fd3
-  data.tar.gz: 83800c9061918d37f3be5a5b4f3137346425328a33f013dbd37dd58f3c818dd4e411605f372aabd0c6a5501a694ddf5928246e96d5e931af5e5468cc2d699777
+  metadata.gz: d29510e81e2ab5510a85360321e77444363531a2786c5f5dd6213514c0f5d97232ac5d19b25eb0fbd3bc9972dc255e326b7f600c2254cdb2fe6ed7be20cd76e9
+  data.tar.gz: 5cbb9e3167a42f86ecd6b93dfd9fe21aca7e599e7faf492e9c8f59b109958b157faff0601f1b8078ee6b3a0e92165eda9cab36d6d8eb9d7c5935f3828da18d74

data/README.md

@@ -46,7 +46,7 @@ adj_table   | boolean | false   | The option is used to give partial credit for
 
 origin formula:
 
-![origin](https://chart.googleapis.com/chart?cht=tx&chl=%5Cbegin%7Bcases%7D%200%20%26%20m%3D0%20%5C%5C%20%5Cfrac%20%7B%201%20%7D%7B%203%20%7D%20(%5Cfrac%20%7B%20m%20%7D%7B%20%5Cleft%7C%20s1%20%5Cright%7C%20%20%7D%20%2B%5Cfrac%20%7B%20m%20%7D%7B%20%5Cleft%7C%20s2%20%5Cright%7C%20%20%7D%20%2B%5Cfrac%20%7B%20m-t%20%7D%7B%20m%20%7D%20)%20%26%20others%20%5Cend%7Bcases%7D)
+![origin](https://chart.googleapis.com/chart?cht=tx&chs&chl=%5Cbegin%7Bcases%7D0%26%7B%5Ctext%7Bif%20%7Dm%3D0%7D%5C%5C%5Cfrac%7B1%7D%7B3%7D(%5Cfrac%7Bm%7D%7B%5Cleft%7Cs1%5Cright%7C%7D%2B%5Cfrac%7Bm%7D%7B%5Cleft%7Cs2%5Cright%7C%7D%2B%5Cfrac%7Bm-t%7D%7Bm%7D)%26%5Ctext%7Bothers%7D%5Cend%7Bcases%7D)
 
 where
 
@@ -55,7 +55,7 @@ where
 
 with adjusting table:
 
-![adj](https://chart.googleapis.com/chart?cht=tx&chl=%5Cbegin%7Bcases%7D%200%20%26%20m%3D0%20%5C%5C%20%5Cfrac%20%7B%201%20%7D%7B%203%20%7D%20(%5Cfrac%20%7B%20%5Cfrac%20%7B%20s%20%7D%7B%2010%20%7D%20%2Bm%20%7D%7B%20%5Cleft%7C%20s1%20%5Cright%7C%20%20%7D%20%2B%5Cfrac%20%7B%20%5Cfrac%20%7B%20s%20%7D%7B%2010%20%7D%20%2Bm%20%7D%7B%20%5Cleft%7C%20s2%20%5Cright%7C%20%20%7D%20%2B%5Cfrac%20%7B%20m-t%20%7D%7B%20m%20%7D%20)%20%26%20others%20%5Cend%7Bcases%7D)
+![adj](https://chart.googleapis.com/chart?cht=tx&chs&chl=%5Cbegin%7Bcases%7D0%26%5Ctext%7Bif%20%7Dm%3D0%5C%5C%5Cfrac%7B1%7D%7B3%7D(%5Cfrac%7B%5Cfrac%7Bs%7D%7B10%7D%2Bm%7D%7B%5Cleft%7Cs1%5Cright%7C%7D%2B%5Cfrac%7B%5Cfrac%7Bs%7D%7B10%7D%2Bm%7D%7B%5Cleft%7Cs2%5Cright%7C%7D%2B%5Cfrac%7Bm-t%7D%7Bm%7D)%26%5Ctext%7Bothers%7D%5Cend%7Bcases%7D)
 
 where
 
@@ -76,7 +76,7 @@ Windows Support | **Yes**      |                  | No          | **Yes**
 Adjusting Table | **Yes**      | No               | No          | No
 Native          | **Yes**      | **Yes**          | **Yes**     | **Yes**
 Pure Ruby       | **Yes**      | **Yes**          | No          | No
-Speed           | Medium       | Fast             | Medium      | Slow
+Speed           | Medium       | **Fast**         | Medium      | Slow
 Bug Found       | **Not Yet**  | Yes              | **Not Yet** | Yes
 
 For `Bug Found`, I made a rake task to build the table below, the source code is in `Rakefile`:
@@ -99,30 +99,28 @@ str_1      | str_2      | origin | jaro_winkler | fuzzystringmatch | hotwater |
 
 ### Pure Ruby
 
-                 | user      | system   | total     | real
----------------- | --------- | -------- | --------- | ------------
-jaro_winkler     | 12.750000 | 0.030000 | 12.780000 | ( 12.782842)
-fuzzystringmatch | 16.240000 | 0.030000 | 16.270000 | ( 16.287380)
+                 | user     | system   | total    | real
+---------------- | -------- | -------- | -------- | ------------
+jaro_winkler     | 1.300000 | 0.000000 | 1.300000 | (  1.299802)
+fuzzystringmatch | 1.510000 | 0.000000 | 1.510000 | (  1.510136)
 
-- jaro_winkler (1.2.3)
+- jaro_winkler (1.3.1)
 - fuzzy-string-match (0.9.6)
 
 ### Native
 
                  | user     | system   | total    | real
 ---------------- | -------- | -------- | -------- | ------------
-jaro_winkler     | 0.390000 | 0.000000 | 0.390000 | (  0.392408)
-fuzzystringmatch | 0.150000 | 0.000000 | 0.150000 | (  0.151552)
-hotwater         | 0.320000 | 0.000000 | 0.320000 | (  0.317740)
-amatch           | 0.960000 | 0.010000 | 0.970000 | (  0.964803)
+jaro_winkler     | 0.350000 | 0.010000 | 0.360000 | (  0.345293)
+fuzzystringmatch | 0.140000 | 0.000000 | 0.140000 | (  0.138711)
+hotwater         | 0.310000 | 0.000000 | 0.310000 | (  0.306498)
+amatch           | 0.960000 | 0.000000 | 0.960000 | (  0.961509)
 
-- jaro_winkler (1.2.3)
+- jaro_winkler (1.3.1)
 - fuzzy-string-match (0.9.6)
 - hotwater (0.1.2)
 - amatch (0.3.0)
 
 # Todo
 
-- Custom adjusting word table.
-- If the adjusting table is ASCII encoded, use dense matrix instread of sparse matrix to speed up.
-- Call by reference instead of call by value to enhance performance.
+- Custom adjusting word table.

data/benchmark/pure.rb

@@ -3,7 +3,7 @@ require 'jaro_winkler'
 require 'fuzzystringmatch'
 ary = [['al', 'al'], ['martha', 'marhta'], ['jones', 'johnson'], ['abcvwxyz', 'cabvwxyz'], ['dwayne', 'duane'], ['dixon', 'dicksonx'], ['fvie', 'ten']]
 
-n = 100000
+n = 10000
 Benchmark.bmbm do |x|
   x.report 'jaro_winkler' do
     n.times{ ary.each{ |str1, str2| JaroWinkler.r_distance(str1, str2) } }

data/benchmark/pure.txt

@@ -1,8 +1,8 @@
 Rehearsal ----------------------------------------------------
-jaro_winkler      12.690000   0.020000  12.710000 ( 12.726929)
-fuzzystringmatch  15.230000   0.020000  15.250000 ( 15.255146)
------------------------------------------- total: 27.960000sec
+jaro_winkler       1.300000   0.000000   1.300000 (  1.300723)
+fuzzystringmatch   1.500000   0.010000   1.510000 (  1.497842)
+------------------------------------------- total: 2.810000sec
 
                        user     system      total        real
-jaro_winkler      12.290000   0.020000  12.310000 ( 12.308876)
-fuzzystringmatch  15.460000   0.020000  15.480000 ( 15.493061)
+jaro_winkler       1.300000   0.000000   1.300000 (  1.299802)
+fuzzystringmatch   1.510000   0.000000   1.510000 (  1.510136)

data/ext/jaro_winkler/adj_matrix.c

@@ -0,0 +1,62 @@
+#include <stdlib.h>
+#include "adj_matrix.h"
+
+extern unsigned int MurmurHash2(const void * key, int len, unsigned int seed);
+static void node_free(Node *head);
+
+AdjMatrix* adj_matrix_new(unsigned int length){
+  AdjMatrix *matrix = malloc(sizeof(AdjMatrix));
+  matrix->length = length == 0 ? ADJ_MATRIX_DEFAULT_LENGTH : length;
+  matrix->table = malloc(matrix->length * sizeof(Node**));
+  for(int i = 0; i < matrix->length; i++){
+    matrix->table[i] = malloc(matrix->length * sizeof(Node*));
+    for (int j = 0; j < matrix->length; j++)
+      matrix->table[i][j] = NULL;
+  }
+  return matrix;
+}
+
+void adj_matrix_add(AdjMatrix *matrix, unsigned long long x, unsigned long long y){
+  unsigned int h1 = MurmurHash2(&x, sizeof(long long), ADJ_MATRIX_SEED) % ADJ_MATRIX_DEFAULT_LENGTH,
+               h2 = MurmurHash2(&y, sizeof(long long), ADJ_MATRIX_SEED) % ADJ_MATRIX_DEFAULT_LENGTH;
+  Node *new_node = malloc(sizeof(Node)); new_node->x = h1; new_node->y = h2; new_node->next = NULL;
+  if(matrix->table[h1][h2] == NULL){
+    matrix->table[h1][h2] = matrix->table[h2][h1] = new_node;
+  }
+  else{
+    Node *previous = NULL;
+    for(Node *i = matrix->table[h1][h2]; i != NULL; i = i->next) previous = i;
+    previous->next = new_node;
+  }
+}
+
+char adj_matrix_find(AdjMatrix *matrix, unsigned long long x, unsigned long long y){
+  unsigned int h1 = MurmurHash2(&x, sizeof(long long), ADJ_MATRIX_SEED) % ADJ_MATRIX_DEFAULT_LENGTH,
+               h2 = MurmurHash2(&y, sizeof(long long), ADJ_MATRIX_SEED) % ADJ_MATRIX_DEFAULT_LENGTH;
+  Node *node = matrix->table[h1][h2];
+  if(node == NULL) return 0;
+  else{
+    for(Node *i = node; i != NULL; i = i->next)
+      if((i->x == h1 && i->y == h2) || (i->x == h2 && i->y == h1)) return 1;
+    return 0;
+  }
+}
+
+static void node_free(Node *head){
+  if(head == NULL) return;
+  node_free(head->next);
+  free(head);
+}
+
+void adj_matrix_free(AdjMatrix *matrix){
+  for(int i = 0; i < matrix->length; i++){
+    for(int j = 0; j < matrix->length; j++)
+      if(matrix->table[i][j] != NULL){
+        node_free(matrix->table[i][j]);
+        matrix->table[i][j] = matrix->table[j][i] = NULL;
+      }
+    free(matrix->table[i]);
+  }
+  free(matrix->table);
+  free(matrix);
+}

data/ext/jaro_winkler/adj_matrix.h

@@ -0,0 +1,21 @@
+#ifndef ADJ_MATRIX_H
+#define ADJ_MATRIX_H 1
+#define ADJ_MATRIX_DEFAULT_LENGTH 958
+#define ADJ_MATRIX_SEED 9527
+
+typedef struct _node{
+  struct _node *next;
+  unsigned long long x, y;
+} Node;
+
+typedef struct{
+  Node ***table;
+  unsigned int length;
+} AdjMatrix;
+
+AdjMatrix* adj_matrix_new(unsigned int length);
+void adj_matrix_add(AdjMatrix *matrix, unsigned long long x, unsigned long long y);
+char adj_matrix_find(AdjMatrix *matrix, unsigned long long x, unsigned long long y);
+void adj_matrix_free(AdjMatrix *matrix);
+
+#endif /* ADJ_MATRIX_H */

data/ext/jaro_winkler/distance.c

@@ -2,6 +2,7 @@
 #include <stdlib.h>
 #include <ctype.h>
 #include "distance.h"
+#include "adj_matrix.h"
 
 typedef struct{
   unsigned long long code;
@@ -13,27 +14,16 @@ typedef struct{
   int length;
 } Codepoints;
 
-typedef struct{
-  unsigned long long x, y;
-} Coord;
-
-typedef struct{
-  Coord *coords;
-  int length;
-} Matrix;
-
-static const char *DEFAULT_ADJ_TABLE[] = {
+const char *DEFAULT_ADJ_TABLE[] = {
   "A","E", "A","I", "A","O", "A","U", "B","V", "E","I", "E","O", "E","U", "I","O", "I","U", "O","U",
   "I","Y", "E","Y", "C","G", "E","F", "W","U", "W","V", "X","K", "S","Z", "X","S", "Q","C", "U","V",
   "M","N", "L","I", "Q","O", "P","R", "I","J", "2","Z", "5","S", "8","B", "1","I", "1","L", "0","O",
   "0","Q", "C","K", "G","J", "E"," ", "Y"," ", "S"," "
 };
-static Matrix DEFAULT_MATRIX;
 
 static UnicodeHash unicode_hash_new(const char *str);
 static Codepoints codepoints_new(const char *str, int byte_len);
-static Matrix matrix_new(const char **adj_table, int length);
-static char matrix_find(Matrix matrix, unsigned long long code_1, unsigned long long code_2);
+static AdjMatrix* adj_matrix_default();
 
 Option option_new(){
   Option opt;
@@ -61,13 +51,6 @@ double c_distance(char *s1, int s1_byte_len, char *s2, int s2_byte_len, Option o
     int tmp2 = code_ary_1.length; code_ary_1.length = code_ary_2.length; code_ary_2.length = tmp2;
   }
 
-  // Adjusting table
-  static char first_time = 1;
-  if(opt.adj_table){
-    if(first_time) DEFAULT_MATRIX = matrix_new(DEFAULT_ADJ_TABLE, sizeof(DEFAULT_ADJ_TABLE) / 8);
-    first_time = 0;
-  }
-
   // Compute jaro distance
   int window_size = code_ary_2.length / 2 - 1;
   if(window_size < 0) window_size = 0;
@@ -91,7 +74,7 @@ double c_distance(char *s1, int s1_byte_len, char *s2, int s2_byte_len, Option o
           previous_index = j;
           found = 1;
         }
-      }else if(opt.adj_table && matrix_find(DEFAULT_MATRIX, code_ary_1.ary[i], code_ary_2.ary[j])) sim_matched = 1;
+      }else if(opt.adj_table && adj_matrix_find(adj_matrix_default(), code_ary_1.ary[i], code_ary_2.ary[j])) sim_matched = 1;
     } // for(int j = left; j <= right; j++){
     if(matched){
       matches++;
@@ -130,7 +113,7 @@ static UnicodeHash unicode_hash_new(const char *str){
 }
 
 static Codepoints codepoints_new(const char *str, int byte_len){
-  Codepoints ret = {0};
+  Codepoints ret = {};
   ret.ary = calloc(byte_len, sizeof(long long));
   int count = 0;
   for(int i = 0; i < byte_len;){
@@ -143,21 +126,16 @@ static Codepoints codepoints_new(const char *str, int byte_len){
   return ret;
 }
 
-static Matrix matrix_new(const char **adj_table, int length){
-  Matrix ret;
-  ret.coords = calloc(length, sizeof(Coord));
-  ret.length = length;
-  for(int i = 0; i < length; i += 2){
-    UnicodeHash h1 = unicode_hash_new(adj_table[i]);
-    UnicodeHash h2 = unicode_hash_new(adj_table[i+1]);
-    ret.coords[i].x = ret.coords[i+1].y = h1.code;
-    ret.coords[i].y = ret.coords[i+1].x = h2.code;
+static AdjMatrix* adj_matrix_default(){
+  static char first_time = 1;
+  static AdjMatrix *ret_matrix;
+  if(first_time){
+    ret_matrix = adj_matrix_new(ADJ_MATRIX_DEFAULT_LENGTH);
+    for(int i = 0; i < 78; i += 2){
+      UnicodeHash h1 = unicode_hash_new(DEFAULT_ADJ_TABLE[i]), h2 = unicode_hash_new(DEFAULT_ADJ_TABLE[i + 1]);
+      adj_matrix_add(ret_matrix, h1.code, h2.code);
+    }
+    first_time = 0;
   }
-  return ret;
-}
-
-static char matrix_find(Matrix matrix, unsigned long long code_1, unsigned long long code_2){
-  for (int i = 0; i < matrix.length; i++)
-    if(matrix.coords[i].x == code_1 && matrix.coords[i].y == code_2) return 1;
-  return 0;
+  return ret_matrix;
 }

data/ext/jaro_winkler/distance.h

@@ -6,7 +6,7 @@ typedef struct{
   char ignore_case, adj_table;
 } Option;
 
-double      c_distance(char *s1, int s1_byte_len, char *s2, int s2_byte_len, Option opt);
-Option      option_new();
+double c_distance(char *s1, int s1_byte_len, char *s2, int s2_byte_len, Option opt);
+Option option_new();
 
 #endif /* DISTANCE_H */

data/ext/jaro_winkler/jaro_winkler.c

@@ -21,8 +21,7 @@ VALUE rb_distance(int argc, VALUE *argv, VALUE self){
     if(c_opt.weight > 0.25) rb_raise(rb_eRuntimeError, "Scaling factor should not exceed 0.25, otherwise the distance can become larger than 1.");
     if(!NIL_P(threshold)) c_opt.threshold     = NUM2DBL(threshold);
     if(!NIL_P(ignore_case)) c_opt.ignore_case = (TYPE(ignore_case)  == T_FALSE || NIL_P(ignore_case)) ? 0 : 1;
-    if(!NIL_P(adj_table)) c_opt.adj_table     = (TYPE(adj_table)    == T_FALSE || NIL_P(adj_table)) ? 0 : 1;
+    if(!NIL_P(adj_table)) c_opt.adj_table     = (TYPE(adj_table)    == T_FALSE || NIL_P(adj_table))   ? 0 : 1;
   }
-  VALUE ret = rb_float_new(c_distance(StringValuePtr(s1), RSTRING_LEN(s1), StringValuePtr(s2), RSTRING_LEN(s2), c_opt));
-  return ret;
+  return rb_float_new(c_distance(StringValuePtr(s1), RSTRING_LEN(s1), StringValuePtr(s2), RSTRING_LEN(s2), c_opt));
 }

data/ext/jaro_winkler/murmur_hash2.c

@@ -0,0 +1,64 @@
+//-----------------------------------------------------------------------------
+// MurmurHash2, by Austin Appleby
+
+// Note - This code makes a few assumptions about how your machine behaves -
+
+// 1. We can read a 4-byte value from any address without crashing
+// 2. sizeof(int) == 4
+
+// And it has a few limitations -
+
+// 1. It will not work incrementally.
+// 2. It will not produce the same results on little-endian and big-endian
+//    machines.
+
+unsigned int MurmurHash2 ( const void * key, int len, unsigned int seed )
+{
+	// 'm' and 'r' are mixing constants generated offline.
+	// They're not really 'magic', they just happen to work well.
+
+	const unsigned int m = 0x5bd1e995;
+	const int r = 24;
+
+	// Initialize the hash to a 'random' value
+
+	unsigned int h = seed ^ len;
+
+	// Mix 4 bytes at a time into the hash
+
+	const unsigned char * data = (const unsigned char *)key;
+
+	while(len >= 4)
+	{
+		unsigned int k = *(unsigned int *)data;
+
+		k *= m; 
+		k ^= k >> r; 
+		k *= m; 
+		
+		h *= m; 
+		h ^= k;
+
+		data += 4;
+		len -= 4;
+	}
+	
+	// Handle the last few bytes of the input array
+
+	switch(len)
+	{
+	case 3: h ^= data[2] << 16;
+	case 2: h ^= data[1] << 8;
+	case 1: h ^= data[0];
+	        h *= m;
+	};
+
+	// Do a few final mixes of the hash to ensure the last few
+	// bytes are well-incorporated.
+
+	h ^= h >> 13;
+	h *= m;
+	h ^= h >> 15;
+
+	return h;
+} 

data/lib/jaro_winkler.rb

@@ -4,7 +4,7 @@ require 'jaro_winkler/jaro_winkler.so' unless JaroWinkler.fallback?
 module JaroWinkler
   module_function
   def jaro_distance s1, s2, options = {}
-    options = {adj_table: false}.merge options
+    options[:adj_table]
     length1, length2 = s1.length, s2.length
     # Guarantee the length order
     if s1.length > s2.length
@@ -52,7 +52,7 @@ module JaroWinkler
   end
 
   def r_distance s1, s2, options = {}
-    options = {weight: 0.1, threshold: 0.7, ignore_case: false}.merge options
+    options = {weight: 0.1, threshold: 0.7, ignore_case: false, adj_table: false}.merge options
     weight, threshold, ignore_case = options[:weight], options[:threshold], options[:ignore_case]
     raise 'Scaling factor should not exceed 0.25, otherwise the distance can become larger than 1' if weight > 0.25
     s1, s2     = s1.upcase, s2.upcase if ignore_case

data/lib/jaro_winkler/version.rb

@@ -1,3 +1,3 @@
 module JaroWinkler
-  VERSION = "1.3.1"
+  VERSION = "1.3.2.beta"
 end

data/spec/jaro_winkler_spec.rb

@@ -55,7 +55,7 @@ shared_examples 'common' do |strategy|
   it 'works with adjusting table' do
     ary = [
       ['HENKA'       , 'HENKAN'      , 0.9667] , # m=5, t=0, s=0
-      ['AL'          , 'AL'          , 1.0   ],  # m=2, t=0, s=0
+      ['AL'          , 'AL'          , 1.0   ] , # m=2, t=0, s=0
       ['MARTHA'      , 'MARHTA'      , 0.9611] , # m=6, t=1, s=0
       ['JONES'       , 'JOHNSON'     , 0.8598] , # m=4, t=0, s=3
       ['ABCVWXYZ'    , 'CABVWXYZ'    , 0.9583] , # m=8, t=1, s=0

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: jaro_winkler
 version: !ruby/object:Gem::Version
-  version: 1.3.1
+  version: 1.3.2.beta
 platform: ruby
 authors:
 - Jian Weihang
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-09-10 00:00:00.000000000 Z
+date: 2014-09-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -72,11 +72,14 @@ files:
 - benchmark/native.txt
 - benchmark/pure.rb
 - benchmark/pure.txt
+- ext/jaro_winkler/adj_matrix.c
+- ext/jaro_winkler/adj_matrix.h
 - ext/jaro_winkler/distance.c
 - ext/jaro_winkler/distance.h
 - ext/jaro_winkler/extconf.rb
 - ext/jaro_winkler/jaro_winkler.c
 - ext/jaro_winkler/jaro_winkler.h
+- ext/jaro_winkler/murmur_hash2.c
 - jaro_winkler.gemspec
 - lib/jaro_winkler.rb
 - lib/jaro_winkler/adjusting_table.rb
@@ -99,9 +102,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - ">"
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
 rubyforge_project: 
 rubygems_version: 2.4.1