jaro_winkler 1.1.1 → 1.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 5ca8266cce395bbcf2b425545e7a426110ef2601
-  data.tar.gz: 23689730155a05387f5908b4661271f20aa118af
+  metadata.gz: 9a1a6510f807b126a00145ca02323746c1885409
+  data.tar.gz: 3cc743dde90a2dc0f682af51b506da4d4dc8e6ed
 SHA512:
-  metadata.gz: 67661a5f12b0204e76cf50fc44bad3a912b3a570602d18402e29065a6d75455cf7666b439f1aecb8b4d8f77b380cc5af8ff0b7c1f143b557b6fb4488198c19cf
-  data.tar.gz: cb172f54e6ac951d068ad805e157d31c233335e50ecf81c89c15d7cdf50b3b092cbfda044b962639b8e0f48d760bcddfb2277893d70a83b018e63cad97084e0f
+  metadata.gz: 02fec80fb44db8e6efed4b6cf083d55e4356cf706827c85850db8ef1c409a483e90c20e6de8437f0282d9621d46513a80061d1dd333d6b4b45b85d81f5c73bd5
+  data.tar.gz: e6e43be13fb520ddfa88206488689f4329c68003b0c4c1c40b69192c6df3d7bcc323f18316f21392ebf0da0dc2fce6949d43b8d8833ad72fe59ca288ebce3132

data/README.md

@@ -20,10 +20,12 @@ JaroWinkler.distance "MARTHA", "MARHTA", weight: 0.2
 # => 0.9778
 
 # Native
-JaroWinkler.c_distance "MARTHA", "MARHTA"
+JaroWinkler.c_distance "MARTHA", "MARHTA" # Recommended one, it's 7 times faster than the latter.
 JaroWinkler.distance "MARTHA", "MARHTA", native: true
 ```
 
+**Both implementations support UTF-8 string.**
+
 ## Options
 
 Name        | Type    | Default | Note
@@ -31,14 +33,7 @@ Name        | Type    | Default | Note
 case_match  | boolean | false   | All upper case characters are converted to lower case prior to the comparison.
 weight      | number  | 0.1     | A constant scaling factor for how much the score is adjusted upwards for having common prefixes.
 threshold   | number  | 0.7     | The prefix bonus is only added when the compared strings have a Jaro distance above a this.
-native      | boolean | false   | Use native version, note that it omits all the other options.
-
-## Pure Ruby v.s. Native
-
-               | Pure | Native
--------------- | ---- | ------
-UTF-8 Support  | Yes  | No
-Option Setting | Yes  | No
+native      | boolean | false   | Use native version.
 
 # Why This?
 
@@ -90,10 +85,5 @@ end
 
 # Todo
 
-- Speed up `#distance(s1, s2, native: true)`
-- Support UTF-8 in native version.
-- Add more optoins to natvie version.
-  - case_match
-  - weight
-  - threshold
-  - adjusting word table (It's from the original C implementation.)
+- Adjusting word table (Reference to original C implementation.)
+- Remove `#c_distance`, use C extension as default, and fallback to Ruby in Java platform.

data/benchmark/native.rb

@@ -20,6 +20,6 @@ Benchmark.bmbm do |x|
 end
 
 #                                       user     system      total        real
-# #c_distance(s1, s2)               0.270000   0.000000   0.270000 (  0.270250)
-# #distance(s1, s2, native: true)   2.030000   0.050000   2.080000 (  2.075878)
-# fuzzystringmatch                  0.140000   0.000000   0.140000 (  0.141239)
+# #c_distance(s1, s2)               0.350000   0.000000   0.350000 (  0.349109)
+# #distance(s1, s2, native: true)   2.480000   0.050000   2.530000 (  2.526027)
+# fuzzystringmatch                  0.160000   0.000000   0.160000 (  0.155539)

data/ext/jaro_winkler/distance.c

@@ -0,0 +1,99 @@
+#include <string.h>
+#include <stdlib.h>
+#include <ctype.h>
+#include "distance.h"
+
+Option* option_new(){
+  Option *opt = calloc(1, sizeof(Option));
+  opt->case_match = 0;
+  opt->weight = 0.1;
+  opt->threshold = 0.7;
+  return opt;
+}
+
+static int char_bytes_num(char first_char){
+  unsigned char c = first_char;
+  if(c >= 252) return 6;      // 1111110x
+  else if(c >= 248) return 5; // 111110xx
+  else if(c >= 240) return 4; // 11110xxx
+  else if(c >= 224) return 3; // 1110xxxx
+  else if(c >= 192) return 2; // 110xxxxx
+  else return 1;
+}
+
+static unsigned long* codepoints(const char *str, int *ret_len){
+  int str_len = strlen(str);
+  unsigned long *ret = calloc(str_len, sizeof(long));
+  int count = 0;
+  for(int i = 0; i < str_len;){
+    int bytes_num = char_bytes_num(str[i]);
+    memcpy(&ret[count], &str[i], bytes_num);
+    count++;
+    i += bytes_num;
+  }
+  *ret_len = count;
+  return ret;
+}
+
+double c_distance(char *s1, char *s2, Option *opt){
+  // set default option if NULL passed
+  int free_opt_flag = 0;
+  if(!opt){ free_opt_flag = 1; opt = option_new(); }
+
+  int ary_1_len, ary_2_len;
+  unsigned long *ary_1 = codepoints(s1, &ary_1_len), *ary_2 = codepoints(s2, &ary_2_len);
+
+  if(opt->case_match){
+    for(int i = 0; i < ary_1_len; ++i) if(ary_1[i] < 256 && islower(ary_1[i])) ary_1[i] -= 32;
+    for(int i = 0; i < ary_2_len; ++i) if(ary_2[i] < 256 && islower(ary_2[i])) ary_2[i] -= 32;
+  }
+
+  // Guarantee the order
+  if(ary_1_len > ary_2_len){
+    unsigned long *tmp = ary_1; ary_1 = ary_2; ary_2 = tmp;
+    int tmp2 = ary_1_len; ary_1_len = ary_2_len; ary_2_len = tmp2;
+  }
+  int window_size = ary_2_len / 2 - 1;
+  if(window_size < 0) window_size = 0;
+  double matches = 0.0;
+  int transpositions = 0;
+  int previous_index = -1;
+  int max_index = ary_2_len - 1;
+  for(int i = 0; i < ary_1_len; i++){
+    int left  = i - window_size;
+    int right = i + window_size;
+    if(left  < 0) left = 0;
+    if(right > max_index) right = max_index;
+    char matched   = 0;
+    char found     = 0;
+    for(int j = left; j <= right; j++){
+      if(ary_1[i] == ary_2[j]){
+        matched = 1;
+        if(!found){
+          if(j > previous_index){
+            previous_index = j;
+            found = 1;
+          }
+        } // if(!found){
+      } // if(ary_1[i] == ary_2[j]){
+    } // for(int j = left; j <= right; j++){
+    if(matched){
+      matches++;
+      if(!found) transpositions++;
+    }
+  } // for(int i = 0; i < ary_1_len; i++){
+  // Don't divide transpositions by 2 since it's been counted directly by above code.
+  double jaro_distance =  matches == 0 ? 0 : (matches / ary_1_len + matches / ary_2_len + (matches - transpositions) / matches) / 3.0;
+
+  // calculate jaro-winkler distance
+  double threshold = opt->threshold, weight = opt->weight;
+  int prefix = 0;
+  int max_length = ary_1_len > 4 ? 4 : ary_1_len;
+  for(int i = 0; i < max_length; ++i){
+    if(ary_1[i] == ary_2[i]) prefix++;
+    else break;
+  }
+  free(ary_1); free(ary_2);
+  if(free_opt_flag) free(opt);
+  return jaro_distance < threshold ? jaro_distance : jaro_distance + ((prefix * weight) * (1 - jaro_distance));
+}

data/ext/jaro_winkler/distance.h

@@ -0,0 +1,12 @@
+#ifndef DISTANCE_H
+#define DISTANCE_H 1
+
+typedef struct{
+  char case_match;
+  double weight, threshold;
+} Option;
+
+double c_distance(char *s1, char *s2, Option *opt);
+Option* option_new();
+
+#endif /* DISTANCE_H */

data/ext/jaro_winkler/jaro_winkler.c

@@ -1,32 +1,27 @@
 #include "jaro_winkler.h"
-#define MAX(X,Y) ((X) < (Y) ? (Y) : (X))
-VALUE rb_mJaroWinkler;
-
-static VALUE distance(VALUE self, VALUE s1, VALUE s2){
-  Check_Type(s1, T_STRING); Check_Type(s2, T_STRING);
-  // Check encoding
-  VALUE s1_ascii_only = TYPE(rb_funcall(s1, rb_intern("ascii_only?"), 0));
-  VALUE s2_ascii_only = TYPE(rb_funcall(s2, rb_intern("ascii_only?"), 0));
-  if(s1_ascii_only == T_FALSE || s2_ascii_only == T_FALSE) printf("WARNING: Non-ASCII string detected.\n");
+#include "distance.h"
 
-  VALUE *s1_ptr = &s1, *s2_ptr = &s2;
-  // guarantee the length of s1_ptr is less than or equal to that of s2_ptr
-  if(RSTRING_LEN(s1) > RSTRING_LEN(s2)){ VALUE *tmp = s1_ptr; s1_ptr = s2_ptr; s2_ptr = tmp; }
-  int min_length = RSTRING_LEN(*s1_ptr), max_length   = RSTRING_LEN(*s2_ptr);
-  char *c_s1_ptr = StringValuePtr(*s1_ptr), *c_s2_ptr = StringValuePtr(*s2_ptr);
-  int opt[] = {1, 0};
-  if(min_length != max_length){
-    // padding spaces
-    char buf[max_length];
-    for(int i = min_length; i < max_length; ++i) buf[i] = ' ';
-    memcpy(buf, c_s1_ptr, min_length);
-    c_s1_ptr = buf;
-    return rb_float_new(strcmp95(c_s1_ptr, c_s2_ptr, max_length, opt));
-  }
-  return rb_float_new(strcmp95(c_s1_ptr, c_s2_ptr, max_length, opt));
-}
+VALUE rb_mJaroWinkler;
 
 void Init_jaro_winkler(void){
   rb_mJaroWinkler = rb_define_module("JaroWinkler");
-  rb_define_module_function(rb_mJaroWinkler, "c_distance", distance, 2);
+  rb_define_module_function(rb_mJaroWinkler, "c_distance", rb_distance, -1);
 }
+
+VALUE rb_distance(int argc, VALUE *argv, VALUE self){
+  VALUE s1, s2, opt;
+  rb_scan_args(argc, argv, "2:", &s1, &s2, &opt);
+  Option *c_opt = option_new();
+  if(TYPE(opt) == T_HASH){
+    VALUE weight = rb_hash_aref(opt, ID2SYM(rb_intern("weight")));
+    VALUE threshold = rb_hash_aref(opt, ID2SYM(rb_intern("threshold")));
+    VALUE case_match = rb_hash_aref(opt, ID2SYM(rb_intern("case_match")));
+    if(!NIL_P(weight)) c_opt->weight = NUM2DBL(weight);
+    if(!NIL_P(threshold)) c_opt->threshold = NUM2DBL(threshold);
+    if(!NIL_P(case_match)) c_opt->case_match = (TYPE(case_match) == T_FALSE || NIL_P(case_match)) ? 0 : 1;
+  }
+  // TODO: replace StringValueCStr with StringValuePtr and RSTRING_LEN
+  VALUE ret = rb_float_new(c_distance(StringValueCStr(s1), StringValueCStr(s2), c_opt));
+  free(c_opt);
+  return ret;
+}

data/ext/jaro_winkler/jaro_winkler.h

@@ -3,6 +3,6 @@
 
 #include "ruby.h"
 
-double strcmp95(char *ying, char *yang, long y_length, int *ind_c);
+VALUE rb_distance(int argc, VALUE *argv, VALUE obj);
 
 #endif /* JARO_WINKLER_H */

data/lib/jaro_winkler.rb

@@ -2,13 +2,14 @@ require 'jaro_winkler/jaro_winkler.so' unless RUBY_PLATFORM == 'java'
 module JaroWinkler
   module_function
   def jaro_distance s1, s2
-    return 0.0 if s1.empty? || s2.empty?
+    length1, length2 = s1.length, s2.length
     # Guarantee the length order
     if s1.length > s2.length
-      tmp = s1; s1 = s2; s2 = tmp
+      s1, s2 = s2, s1
+      length1, length2 = length2, length1
     end
-    length1, length2 = s1.length, s2.length
-    window_size      = (s2.length / 2) - 1
+    window_size      = (length2 / 2) - 1
+    window_size = 0 if window_size < 0
     matches          = 0.0
     transpositions   = 0
     previous_index   = -1
@@ -43,7 +44,7 @@ module JaroWinkler
 
   def distance s1, s2, options = {}
     options = {weight: 0.1, threshold: 0.7, case_match: false, native: false}.merge options
-    return c_distance(s1, s2) if RUBY_PLATFORM != 'java' && options[:native]
+    return c_distance(s1, s2, options) if RUBY_PLATFORM != 'java' && options[:native]
     weight, threshold, case_match = options[:weight], options[:threshold], options[:case_match]
     raise 'Scaling factor should not exceed 0.25, otherwise the distance can become larger than 1' if weight > 0.25
     s1, s2 = s1.downcase, s2.downcase if case_match

data/lib/jaro_winkler/version.rb

@@ -1,3 +1,3 @@
 module JaroWinkler
-  VERSION = "1.1.1"
+  VERSION = "1.2.0"
 end

data/spec/jaro_winkler_spec.rb

@@ -13,6 +13,7 @@ describe JaroWinkler do
       ['fvie', 'ten', 0.0],
       ['tony', 'tony', 1.0],
       ['tonytonyjan', 'tonytonyjan', 1.0],
+      ['x', 'x', 1.0],
       ['', '', 0.0],
       ['tony', '', 0.0],
       ['', 'tony', 0.0],
@@ -29,16 +30,22 @@ describe JaroWinkler do
 
   it 'supports C extension' do
     @ary.each do |s1, s2, ans|
-      expect(distance(s1, s2, native: true)).to be_within(0.0001).of(ans)
+      expect(c_distance(s1, s2)).to be_within(0.0001).of(ans)
     end
   end
 
+  it 'works with UTF-8' do
+    expect(distance('變形金剛4:絕跡重生', '變形金剛4: 絕跡重生')).to eq c_distance('0123456789', '01234x56789')
+  end
+
   it 'can ignore case' do
     expect(distance('MARTHA', 'marhta', case_match: true)).to be_within(0.0001).of(0.9611)
+    expect(c_distance('MARTHA', 'marhta', case_match: true)).to be_within(0.0001).of(0.9611)
   end
 
   it 'can set weight' do
     expect(distance('MARTHA', 'MARHTA', weight: 0.2)).to be_within(0.0001).of(0.9778)
+    expect(c_distance('MARTHA', 'MARHTA', weight: 0.2)).to be_within(0.0001).of(0.9778)
     expect{ distance('MARTHA', 'MARHTA', weight: 0.26) }.to raise_error
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: jaro_winkler
 version: !ruby/object:Gem::Version
-  version: 1.1.1
+  version: 1.2.0
 platform: ruby
 authors:
 - Jian Weihang
@@ -68,6 +68,8 @@ files:
 - Rakefile
 - benchmark/native.rb
 - benchmark/pure.rb
+- ext/jaro_winkler/distance.c
+- ext/jaro_winkler/distance.h
 - ext/jaro_winkler/extconf.rb
 - ext/jaro_winkler/jaro_winkler.c
 - ext/jaro_winkler/jaro_winkler.h