jaro_winkler 1.4.0 → 1.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: fe11a8c8be8cadef3f1f14f847c40a0a8e41268f
-  data.tar.gz: f509aaa81627917a4cf1017884f43b60dcd5483b
+  metadata.gz: 445cf12b6f4c0a9ed5d7bfe9a67ef1e690f9af3c
+  data.tar.gz: b5b7114e72603c057be0d9914c150715a6054670
 SHA512:
-  metadata.gz: 88fca6ab51c8a830de64e23b96f7efaaeeac01cd71735dd3820fc9e84e5de0027872014a8aef0278658d45455a7719a81601604929d17228fd6e8b6cb8fba919
-  data.tar.gz: ab7ee1020506ba6693e2b9878afddcb4a39bf7fb0be89b3958cfc5571f051b8295898cf050d517aa2c1951b23093517ce2d82d8d9c5544e816a4ac9e34c5bc6c
+  metadata.gz: 4d2bfc3518cfbffc5cf68b73f66bade243a1da8ea9f9b0d59b5258514d7fc869c493b2f5f5fdfdf95bd2cc243fdb7c1e67f20f190023c0870d9a3a1facee4993
+  data.tar.gz: 82859b43a0c06fe0a535ac1fdaa07343d7e54dd9a54fb020f0f66930c779cc6f36d78708686fe3c97b62b68d4d317a074fe449cc938eff5f4866ff7406d102ba

data/ext/jaro_winkler/adj_matrix.c

@@ -1,66 +1,75 @@
 #include "adj_matrix.h"
-#include "code.h"
-
-#include <stdlib.h>
+#include "codepoints.h"
+#include "ruby.h"
 
 const char *DEFAULT_ADJ_TABLE[] = {
-  "A","E", "A","I", "A","O", "A","U", "B","V", "E","I", "E","O", "E","U", "I","O", "I","U", "O","U",
-  "I","Y", "E","Y", "C","G", "E","F", "W","U", "W","V", "X","K", "S","Z", "X","S", "Q","C", "U","V",
-  "M","N", "L","I", "Q","O", "P","R", "I","J", "2","Z", "5","S", "8","B", "1","I", "1","L", "0","O",
-  "0","Q", "C","K", "G","J", "E"," ", "Y"," ", "S"," "
-};
+    "A", "E", "A", "I", "A", "O", "A", "U", "B", "V", "E", "I", "E",
+    "O", "E", "U", "I", "O", "I", "U", "O", "U", "I", "Y", "E", "Y",
+    "C", "G", "E", "F", "W", "U", "W", "V", "X", "K", "S", "Z", "X",
+    "S", "Q", "C", "U", "V", "M", "N", "L", "I", "Q", "O", "P", "R",
+    "I", "J", "2", "Z", "5", "S", "8", "B", "1", "I", "1", "L", "0",
+    "O", "0", "Q", "C", "K", "G", "J", "E", " ", "Y", " ", "S", " "};
 
-extern unsigned int MurmurHash2(const void * key, int len, unsigned int seed);
 void node_free(Node *head);
 
-AdjMatrix* adj_matrix_new(unsigned int length){
+AdjMatrix *adj_matrix_new(uint32_t length) {
   AdjMatrix *matrix = malloc(sizeof(AdjMatrix));
   matrix->length = length == 0 ? ADJ_MATRIX_DEFAULT_LENGTH : length;
-  matrix->table = malloc(matrix->length * sizeof(Node**));
-  for(int i = 0; i < matrix->length; i++){
-    matrix->table[i] = malloc(matrix->length * sizeof(Node*));
-    for (int j = 0; j < matrix->length; j++)
+  matrix->table = malloc(matrix->length * sizeof(Node **));
+  for (size_t i = 0; i < matrix->length; i++) {
+    matrix->table[i] = malloc(matrix->length * sizeof(Node *));
+    for (size_t j = 0; j < matrix->length; j++)
       matrix->table[i][j] = NULL;
   }
   return matrix;
 }
 
-void adj_matrix_add(AdjMatrix *matrix, unsigned long long x, unsigned long long y){
-  unsigned int h1 = MurmurHash2(&x, sizeof(long long), ADJ_MATRIX_SEED) % ADJ_MATRIX_DEFAULT_LENGTH,
-               h2 = MurmurHash2(&y, sizeof(long long), ADJ_MATRIX_SEED) % ADJ_MATRIX_DEFAULT_LENGTH;
-  Node *new_node = malloc(sizeof(Node)); new_node->x = h1; new_node->y = h2; new_node->next = NULL;
-  if(matrix->table[h1][h2] == NULL){
+void adj_matrix_add(AdjMatrix *matrix, uint64_t x, uint64_t y) {
+  uint32_t h1 = st_hash(&x, sizeof(x), ADJ_MATRIX_SEED) %
+                ADJ_MATRIX_DEFAULT_LENGTH,
+           h2 = st_hash(&y, sizeof(y), ADJ_MATRIX_SEED) %
+                ADJ_MATRIX_DEFAULT_LENGTH;
+  Node *new_node = malloc(sizeof(Node));
+  new_node->x = h1;
+  new_node->y = h2;
+  new_node->next = NULL;
+  if (matrix->table[h1][h2] == NULL) {
     matrix->table[h1][h2] = matrix->table[h2][h1] = new_node;
-  }
-  else{
+  } else {
     Node *previous = NULL;
-    for(Node *i = matrix->table[h1][h2]; i != NULL; i = i->next) previous = i;
+    for (Node *i = matrix->table[h1][h2]; i != NULL; i = i->next)
+      previous = i;
     previous->next = new_node;
   }
 }
 
-char adj_matrix_find(AdjMatrix *matrix, unsigned long long x, unsigned long long y){
-  unsigned int h1 = MurmurHash2(&x, sizeof(long long), ADJ_MATRIX_SEED) % ADJ_MATRIX_DEFAULT_LENGTH,
-               h2 = MurmurHash2(&y, sizeof(long long), ADJ_MATRIX_SEED) % ADJ_MATRIX_DEFAULT_LENGTH;
+char adj_matrix_find(AdjMatrix *matrix, uint64_t x, uint64_t y) {
+  uint32_t h1 = st_hash(&x, sizeof(x), ADJ_MATRIX_SEED) %
+                ADJ_MATRIX_DEFAULT_LENGTH,
+           h2 = st_hash(&y, sizeof(y), ADJ_MATRIX_SEED) %
+                ADJ_MATRIX_DEFAULT_LENGTH;
   Node *node = matrix->table[h1][h2];
-  if(node == NULL) return 0;
-  else{
-    for(Node *i = node; i != NULL; i = i->next)
-      if((i->x == h1 && i->y == h2) || (i->x == h2 && i->y == h1)) return 1;
+  if (node == NULL)
+    return 0;
+  else {
+    for (Node *i = node; i != NULL; i = i->next)
+      if ((i->x == h1 && i->y == h2) || (i->x == h2 && i->y == h1))
+        return 1;
     return 0;
   }
 }
 
-void node_free(Node *head){
-  if(head == NULL) return;
+void node_free(Node *head) {
+  if (head == NULL)
+    return;
   node_free(head->next);
   free(head);
 }
 
-void adj_matrix_free(AdjMatrix *matrix){
-  for(int i = 0; i < matrix->length; i++){
-    for(int j = 0; j < matrix->length; j++)
-      if(matrix->table[i][j] != NULL){
+void adj_matrix_free(AdjMatrix *matrix) {
+  for (size_t i = 0; i < matrix->length; i++) {
+    for (size_t j = 0; j < matrix->length; j++)
+      if (matrix->table[i][j] != NULL) {
         node_free(matrix->table[i][j]);
         matrix->table[i][j] = matrix->table[j][i] = NULL;
       }
@@ -70,20 +79,20 @@ void adj_matrix_free(AdjMatrix *matrix){
   free(matrix);
 }
 
-AdjMatrix* adj_matrix_default(){
+AdjMatrix *adj_matrix_default() {
   static char first_time = 1;
   static AdjMatrix *ret_matrix;
-  if(first_time){
+  if (first_time) {
     ret_matrix = adj_matrix_new(ADJ_MATRIX_DEFAULT_LENGTH);
-    int length = sizeof(DEFAULT_ADJ_TABLE) / sizeof(char*);
-    for(int i = 0; i < length; i += 2){
-      unsigned long long code_1, code_2;
-      int dummy_length;
-      utf_char_to_code((char*)DEFAULT_ADJ_TABLE[i], &code_1, &dummy_length);
-      utf_char_to_code((char*)DEFAULT_ADJ_TABLE[i+1], &code_2, &dummy_length);
+    size_t length = sizeof(DEFAULT_ADJ_TABLE) / sizeof(char *);
+    for (size_t i = 0; i < length; i += 2) {
+      uint64_t code_1, code_2;
+      size_t dummy_length;
+      code_1 = *DEFAULT_ADJ_TABLE[i] & 0xff;
+      code_2 = *DEFAULT_ADJ_TABLE[i + 1] & 0xff;
       adj_matrix_add(ret_matrix, code_1, code_2);
     }
     first_time = 0;
   }
   return ret_matrix;
-}
+}

data/ext/jaro_winkler/adj_matrix.h

@@ -1,22 +1,22 @@
-#ifndef ADJ_MATRIX_H
-#define ADJ_MATRIX_H
+#pragma once
+
+#include "stdint.h"
+
 #define ADJ_MATRIX_DEFAULT_LENGTH 958
 #define ADJ_MATRIX_SEED 9527
 
-typedef struct _node{
+typedef struct _node {
   struct _node *next;
-  unsigned long long x, y;
+  uint64_t x, y;
 } Node;
 
-typedef struct{
+typedef struct {
   Node ***table;
-  unsigned int length;
+  uint32_t length;
 } AdjMatrix;
 
-AdjMatrix* adj_matrix_new    (unsigned int length);
-void       adj_matrix_add    (AdjMatrix *matrix, unsigned long long x, unsigned long long y);
-char       adj_matrix_find   (AdjMatrix *matrix, unsigned long long x, unsigned long long y);
-void       adj_matrix_free   (AdjMatrix *matrix);
-AdjMatrix* adj_matrix_default();
-
-#endif
+AdjMatrix *adj_matrix_new(uint32_t length);
+void adj_matrix_add(AdjMatrix *matrix, uint64_t x, uint64_t y);
+char adj_matrix_find(AdjMatrix *matrix, uint64_t x, uint64_t y);
+void adj_matrix_free(AdjMatrix *matrix);
+AdjMatrix *adj_matrix_default();

data/ext/jaro_winkler/codepoints.c

@@ -0,0 +1,61 @@
+#include "codepoints.h"
+#include "ruby.h"
+#include "ruby/encoding.h"
+#include <stdint.h>
+#include <stdlib.h>
+#include <string.h>
+
+// this function is copied from string.c
+static inline int single_byte_optimizable(VALUE str) {
+  rb_encoding *enc;
+
+  /* Conservative.  It may be ENC_CODERANGE_UNKNOWN. */
+  if (ENC_CODERANGE(str) == ENC_CODERANGE_7BIT)
+    return 1;
+
+  enc = rb_enc_get(str);
+  if (rb_enc_mbmaxlen(enc) == 1)
+    return 1;
+
+  /* Conservative.  Possibly single byte.
+   * "\xa1" in Shift_JIS for example. */
+  return 0;
+}
+
+void codepoints_init(CodePoints *codepoints, VALUE str) {
+  size_t i, length;
+  int32_t n;
+  uint32_t c;
+  const char *ptr, *end;
+  rb_encoding *enc;
+
+  if (single_byte_optimizable(str)) {
+    length = RSTRING_LEN(str);
+    ptr = RSTRING_PTR(str);
+    codepoints->data = malloc(length * sizeof(*codepoints->data));
+    for (i = 0, codepoints->length = 0; i < length; i++, codepoints->length++)
+      codepoints->data[i] = ptr[i] & 0xff;
+  } else {
+    codepoints->length = 0;
+    codepoints->size = 32;
+    codepoints->data = malloc(codepoints->size * sizeof(*codepoints->data));
+    str = rb_str_new_frozen(str);
+    ptr = RSTRING_PTR(str);
+    end = RSTRING_END(str);
+    enc = rb_enc_get(str);
+
+    while (ptr < end) {
+      c = rb_enc_codepoint_len(ptr, end, &n, enc);
+      if (codepoints->length == codepoints->size) {
+        codepoints->size *= 2;
+        codepoints->data = realloc(codepoints->data, sizeof(*codepoints->data) *
+                                                         codepoints->size);
+      }
+      codepoints->data[codepoints->length++] = c;
+      ptr += n;
+    }
+    RB_GC_GUARD(str);
+  }
+}
+
+void codepoints_free(CodePoints *codepoints) { free(codepoints->data); }

data/ext/jaro_winkler/codepoints.h

@@ -0,0 +1,13 @@
+#pragma once
+#include "ruby.h"
+#include <stddef.h>
+#include <stdint.h>
+
+typedef struct {
+  uint32_t *data;
+  size_t length;
+  size_t size;
+} CodePoints;
+
+void codepoints_init(CodePoints *, VALUE str);
+void codepoints_free(CodePoints *);

data/ext/jaro_winkler/extconf.rb

@@ -1,11 +1,3 @@
-unless RUBY_PLATFORM == 'java'
-  require 'mkmf'
-  $CFLAGS << ' -std=c99 '
-  create_makefile('jaro_winkler/jaro_winkler_ext')
-else
-  dummy_makefile = open('Makefile', 'wb')
-  dummy_makefile.puts '.PHONY: install'
-  dummy_makefile.puts 'install:'
-  dummy_makefile.puts "\t" + '@echo "C extension is not installed, fall back to pure Ruby version instead."'
-  dummy_makefile.close
-end
+require 'mkmf'
+$CFLAGS << ' -std=c99 '
+create_makefile('jaro_winkler/jaro_winkler_ext')

data/ext/jaro_winkler/jaro.c

@@ -1,73 +1,62 @@
 #include "jaro.h"
-#include "code.h"
 #include "adj_matrix.h"
+#include "codepoints.h"
 
-#include <string.h>
-#include <stdlib.h>
 #include <ctype.h>
+#include <stdlib.h>
+#include <string.h>
 
-#define SWAP(x, y) do{ __typeof__(x) SWAP = x; x = y; y = SWAP; }while(0)
-
-double jaro_distance_from_codes(unsigned long long *codes1, int len1, unsigned long long *codes2, int len2, LibJaroOption *opt);
-double jaro_winkler_distance_from_codes(unsigned long long *codes1, int len1, unsigned long long *codes2, int len2, LibJaroOption *opt);
-
-double jaro_distance(char* short_str, int short_str_len, char* long_str, int long_str_len, LibJaroOption *opt){
-  if(!short_str_len || !long_str_len) return 0.0;
-
-  unsigned long long *short_codes, *long_codes;
-  int short_codes_len, long_codes_len;
-  string_to_codes(short_str, short_str_len, &short_codes, &short_codes_len);
-  string_to_codes(long_str, long_str_len, &long_codes, &long_codes_len);
-
-  double ret = jaro_distance_from_codes(short_codes, short_codes_len, long_codes, long_codes_len, opt);
-
-  free(short_codes); free(long_codes);
-  return ret;
-}
-
-double jaro_winkler_distance(char* short_str, int short_str_len, char* long_str, int long_str_len, LibJaroOption *opt){
-  if(!short_str_len || !long_str_len) return 0.0;
-
-  unsigned long long *short_codes, *long_codes;
-  int short_codes_len, long_codes_len;
-  string_to_codes(short_str, short_str_len, &short_codes, &short_codes_len);
-  string_to_codes(long_str, long_str_len, &long_codes, &long_codes_len);
-
-  double ret = jaro_winkler_distance_from_codes(short_codes, short_codes_len, long_codes, long_codes_len, opt);
-
-  free(short_codes); free(long_codes);
-  return ret;
-}
-
-double jaro_distance_from_codes(unsigned long long* short_codes, int short_codes_len, unsigned long long* long_codes, int long_codes_len, LibJaroOption *opt){
-  if(!short_codes_len || !long_codes_len) return 0.0;
-
-  if(short_codes_len > long_codes_len){
-    SWAP(short_codes, long_codes);
-    SWAP(short_codes_len, long_codes_len);
+#define DEFAULT_WEIGHT 0.1
+#define DEFAULT_THRESHOLD 0.7
+#define SWAP(x, y)                                                             \
+  do {                                                                         \
+    __typeof__(x) SWAP = x;                                                    \
+    x = y;                                                                     \
+    y = SWAP;                                                                  \
+  } while (0)
+
+const Options DEFAULT_OPTIONS = {.weight = DEFAULT_WEIGHT,
+                                 .threshold = DEFAULT_THRESHOLD,
+                                 .ignore_case = 0,
+                                 .adj_table = 0};
+
+double jaro_distance_from_codes(uint32_t *codepoints1, size_t len1,
+                                uint32_t *codepoints2, size_t len2,
+                                Options *opt) {
+  if (!len1 || !len2)
+    return 0.0;
+
+  if (len1 > len2) {
+    SWAP(codepoints1, codepoints2);
+    SWAP(len1, len2);
   }
 
-  if(opt->ignore_case){
-    for(int i = 0; i < short_codes_len; i++) short_codes[i] = tolower(short_codes[i]);
-    for(int i = 0; i < long_codes_len; i++) long_codes[i] = tolower(long_codes[i]);
+  if (opt->ignore_case) {
+    for (size_t i = 0; i < len1; i++)
+      codepoints1[i] = tolower(codepoints1[i]);
+    for (size_t i = 0; i < len2; i++)
+      codepoints2[i] = tolower(codepoints2[i]);
   }
 
-  int window_size = long_codes_len/2 - 1;
-  if(window_size < 0) window_size = 0;
+  int32_t window_size = len2 / 2 - 1;
+  if (window_size < 0)
+    window_size = 0;
 
-  char short_codes_flag[short_codes_len];
-  char long_codes_flag[long_codes_len];
-  memset(short_codes_flag, 0, short_codes_len);
-  memset(long_codes_flag, 0, long_codes_len);
+  char short_codes_flag[len1];
+  char long_codes_flag[len2];
+  memset(short_codes_flag, 0, len1);
+  memset(long_codes_flag, 0, len2);
 
   // count number of matching characters
-  int match_count = 0;
-  for(int i = 0; i < short_codes_len; i++){
-    int left = (i >= window_size) ? i - window_size : 0;
-    int right = (i + window_size <= long_codes_len - 1) ? (i + window_size) : (long_codes_len - 1);
-    if(right > long_codes_len - 1) right = long_codes_len - 1;
-    for(int j = left; j <= right; j++){
-      if(!long_codes_flag[j] && short_codes[i] == long_codes[j]){
+  size_t match_count = 0;
+  for (size_t i = 0; i < len1; i++) {
+    size_t left = (i >= window_size) ? i - window_size : 0;
+    size_t right =
+        (i + window_size <= len2 - 1) ? (i + window_size) : (len2 - 1);
+    if (right > len2 - 1)
+      right = len2 - 1;
+    for (size_t j = left; j <= right; j++) {
+      if (!long_codes_flag[j] && codepoints1[i] == codepoints2[j]) {
         short_codes_flag[i] = long_codes_flag[j] = 1;
         match_count++;
         break;
@@ -75,48 +64,58 @@ double jaro_distance_from_codes(unsigned long long* short_codes, int short_codes
     }
   }
 
-  if(!match_count) return 0.0;
+  if (!match_count)
+    return 0.0;
 
   // count number of transpositions
-  int transposition_count = 0, j = 0, k = 0;
-  for(int i = 0; i < short_codes_len; i++){
-    if(short_codes_flag[i]){
-      for(j = k; j < long_codes_len; j++){
-        if(long_codes_flag[j]){
+  size_t transposition_count = 0, j = 0, k = 0;
+  for (size_t i = 0; i < len1; i++) {
+    if (short_codes_flag[i]) {
+      for (j = k; j < len2; j++) {
+        if (long_codes_flag[j]) {
           k = j + 1;
           break;
         }
       }
-      if(short_codes[i] != long_codes[j]) transposition_count++;
+      if (codepoints1[i] != codepoints2[j])
+        transposition_count++;
     }
   }
 
   // count similarities in nonmatched characters
-  int similar_count = 0;
-  if(opt->adj_table && short_codes_len > match_count)
-    for(int i = 0; i < short_codes_len; i++)
-      if(!short_codes_flag[i])
-        for(int j = 0; j < long_codes_len; j++)
-          if(!long_codes_flag[j])
-            if(adj_matrix_find(adj_matrix_default(), short_codes[i], long_codes[j])){
+  size_t similar_count = 0;
+  if (opt->adj_table && len1 > match_count)
+    for (size_t i = 0; i < len1; i++)
+      if (!short_codes_flag[i])
+        for (size_t j = 0; j < len2; j++)
+          if (!long_codes_flag[j])
+            if (adj_matrix_find(adj_matrix_default(), codepoints1[i],
+                                codepoints2[j])) {
               similar_count += 3;
               break;
             }
 
   double m = (double)match_count;
-  double t = (double)(transposition_count/2);
-  if(opt->adj_table) m = similar_count/10.0 + m;
-  return (m/short_codes_len + m/long_codes_len + (m-t)/m) / 3;
+  double t = (double)(transposition_count / 2);
+  if (opt->adj_table)
+    m = similar_count / 10.0 + m;
+  return (m / len1 + m / len2 + (m - t) / m) / 3;
 }
 
-double jaro_winkler_distance_from_codes(unsigned long long* short_codes, int short_codes_len, unsigned long long* long_codes, int long_codes_len, LibJaroOption *opt){
-  double jaro_distance = jaro_distance_from_codes(short_codes, short_codes_len, long_codes, long_codes_len, opt);
-
-  if(jaro_distance < opt->threshold) return jaro_distance;
-  else{
-    int prefix = 0;
-    int max_4 = short_codes_len > 4 ? 4 : short_codes_len;
-    for(prefix = 0; prefix < max_4 && short_codes[prefix] == long_codes[prefix]; prefix++);
-    return jaro_distance + prefix*opt->weight*(1-jaro_distance);
+double jaro_winkler_distance_from_codes(uint32_t *codepoints1, size_t len1,
+                                        uint32_t *codepoints2, size_t len2,
+                                        Options *opt) {
+  double jaro_distance =
+      jaro_distance_from_codes(codepoints1, len1, codepoints2, len2, opt);
+
+  if (jaro_distance < opt->threshold)
+    return jaro_distance;
+  else {
+    size_t prefix = 0;
+    size_t max_4 = len1 > 4 ? 4 : len1;
+    for (prefix = 0;
+         prefix < max_4 && codepoints1[prefix] == codepoints2[prefix]; prefix++)
+      ;
+    return jaro_distance + prefix * opt->weight * (1 - jaro_distance);
   }
-}
+}

data/ext/jaro_winkler/jaro.h

@@ -1,17 +1,17 @@
-#ifndef LIBJARO_JARO_H
-#define LIBJARO_JARO_H
+#pragma once
 
-#define DEFAULT_WEIGHT 0.1
-#define DEFAULT_THRESHOLD 0.7
+#include <stddef.h>
+#include <stdint.h>
 
-typedef struct LibJaroOption{
+typedef struct {
   double weight, threshold;
   char ignore_case, adj_table;
-} LibJaroOption;
+} Options;
 
+extern const Options DEFAULT_OPTIONS;
 
-static const LibJaroOption DEFAULT_OPT = {.weight = DEFAULT_WEIGHT, .threshold = DEFAULT_THRESHOLD, .ignore_case = 0, .adj_table = 0};
-double jaro_distance(char *str1, int len1, char *str2, int len2, LibJaroOption *opt);
-double jaro_winkler_distance(char *str1, int len1, char *str2, int len2, LibJaroOption *opt);
-
-#endif
+double jaro_distance_from_codes(uint32_t *codepoints1, size_t len1,
+                                uint32_t *codepoints2, size_t len2, Options *);
+double jaro_winkler_distance_from_codes(uint32_t *codepoints1, size_t len1,
+                                        uint32_t *codepoints2, size_t len2,
+                                        Options *);

data/ext/jaro_winkler/jaro_winkler.c

@@ -1,45 +1,70 @@
-#include "ruby.h"
+#include "codepoints.h"
 #include "jaro.h"
+#include "ruby.h"
 
-VALUE rb_mJaroWinkler,
-      rb_eError,
-      rb_eInvalidWeightError;
+VALUE rb_mJaroWinkler, rb_eError, rb_eInvalidWeightError;
 
-VALUE rb_jaro_winkler_distance(int argc, VALUE *argv, VALUE self);
-VALUE rb_jaro_distance(int argc, VALUE *argv, VALUE self);
-VALUE distance(int argc, VALUE *argv, VALUE self, double (*distance_fn)(char *str1, int len1, char *str2, int len2, LibJaroOption *opt));
+VALUE rb_jaro_winkler_distance(size_t argc, VALUE *argv, VALUE self);
+VALUE rb_jaro_distance(size_t argc, VALUE *argv, VALUE self);
+VALUE distance(size_t argc, VALUE *argv, VALUE self,
+               double (*distance_fn)(uint32_t *codepoints1, size_t len1,
+                                     uint32_t *codepoints2, size_t len2,
+                                     Options *));
 
-void Init_jaro_winkler_ext(void){
+void Init_jaro_winkler_ext(void) {
   rb_mJaroWinkler = rb_define_module("JaroWinkler");
   rb_eError = rb_define_class_under(rb_mJaroWinkler, "Error", rb_eRuntimeError);
-  rb_eInvalidWeightError = rb_define_class_under(rb_mJaroWinkler, "InvalidWeightError", rb_eError);
-  rb_define_module_function(rb_mJaroWinkler, "distance", rb_jaro_winkler_distance, -1);
-  rb_define_module_function(rb_mJaroWinkler, "jaro_distance", rb_jaro_distance, -1);
+  rb_eInvalidWeightError =
+      rb_define_class_under(rb_mJaroWinkler, "InvalidWeightError", rb_eError);
+  rb_define_singleton_method(rb_mJaroWinkler, "distance",
+                             rb_jaro_winkler_distance, -1);
+  rb_define_singleton_method(rb_mJaroWinkler, "jaro_distance", rb_jaro_distance,
+                             -1);
 }
 
-
-VALUE distance(int argc, VALUE *argv, VALUE self, double (*distance_fn)(char *str1, int len1, char *str2, int len2, LibJaroOption *opt)){
+VALUE distance(size_t argc, VALUE *argv, VALUE self,
+               double (*distance_fn)(uint32_t *codepoints1, size_t len1,
+                                     uint32_t *codepoints2, size_t len2,
+                                     Options *)) {
   VALUE s1, s2, opt;
+  CodePoints cp1, cp2;
+
   rb_scan_args(argc, argv, "2:", &s1, &s2, &opt);
-  LibJaroOption c_opt = DEFAULT_OPT;
-  if(TYPE(opt) == T_HASH){
+  codepoints_init(&cp1, s1);
+  codepoints_init(&cp2, s2);
+
+  Options c_opt = DEFAULT_OPTIONS;
+  if (TYPE(opt) == T_HASH) {
     VALUE weight = rb_hash_aref(opt, ID2SYM(rb_intern("weight"))),
           threshold = rb_hash_aref(opt, ID2SYM(rb_intern("threshold"))),
           ignore_case = rb_hash_aref(opt, ID2SYM(rb_intern("ignore_case"))),
           adj_table = rb_hash_aref(opt, ID2SYM(rb_intern("adj_table")));
-    if(!NIL_P(weight)) c_opt.weight = NUM2DBL(weight);
-    if(c_opt.weight > 0.25) rb_raise(rb_eInvalidWeightError, "Scaling factor should not exceed 0.25, otherwise the distance can become larger than 1.");
-    if(!NIL_P(threshold)) c_opt.threshold = NUM2DBL(threshold);
-    if(!NIL_P(ignore_case)) c_opt.ignore_case = (TYPE(ignore_case) == T_FALSE || NIL_P(ignore_case)) ? 0 : 1;
-    if(!NIL_P(adj_table)) c_opt.adj_table = (TYPE(adj_table) == T_FALSE || NIL_P(adj_table)) ? 0 : 1;
+    if (!NIL_P(weight))
+      c_opt.weight = NUM2DBL(weight);
+    if (c_opt.weight > 0.25)
+      rb_raise(rb_eInvalidWeightError, "Scaling factor should not exceed 0.25, "
+                                       "otherwise the distance can become "
+                                       "larger than 1.");
+    if (!NIL_P(threshold))
+      c_opt.threshold = NUM2DBL(threshold);
+    if (!NIL_P(ignore_case))
+      c_opt.ignore_case =
+          (TYPE(ignore_case) == T_FALSE || NIL_P(ignore_case)) ? 0 : 1;
+    if (!NIL_P(adj_table))
+      c_opt.adj_table =
+          (TYPE(adj_table) == T_FALSE || NIL_P(adj_table)) ? 0 : 1;
   }
-  return rb_float_new((*distance_fn)(StringValuePtr(s1), RSTRING_LEN(s1), StringValuePtr(s2), RSTRING_LEN(s2), &c_opt));
+  VALUE ret = rb_float_new(
+      (*distance_fn)(cp1.data, cp1.length, cp2.data, cp2.length, &c_opt));
+  codepoints_free(&cp1);
+  codepoints_free(&cp2);
+  return ret;
 }
 
-VALUE rb_jaro_distance(int argc, VALUE *argv, VALUE self){
-  return distance(argc, argv, self, jaro_distance);
+VALUE rb_jaro_distance(size_t argc, VALUE *argv, VALUE self) {
+  return distance(argc, argv, self, jaro_distance_from_codes);
 }
 
-VALUE rb_jaro_winkler_distance(int argc, VALUE *argv, VALUE self){
-  return distance(argc, argv, self, jaro_winkler_distance);
-}
+VALUE rb_jaro_winkler_distance(size_t argc, VALUE *argv, VALUE self) {
+  return distance(argc, argv, self, jaro_winkler_distance_from_codes);
+}

data/lib/jaro_winkler.rb

@@ -1,9 +1,9 @@
+# frozen_string_literal: true
+
 require 'jaro_winkler/version'
 
-case RUBY_PLATFORM
-when 'java'
-  require 'jaro_winkler/jaro_winkler_pure'
-else
+if RUBY_ENGINE == 'ruby'
   require 'jaro_winkler/jaro_winkler_ext'
+else
+  require 'jaro_winkler/jaro_winkler_pure'
 end
-  

data/lib/jaro_winkler/adjusting_table.rb

@@ -1,19 +1,14 @@
+# frozen_string_literal: true
+
 module JaroWinkler
-  DEFAULT_ADJ_TABLE = Hash.new
+  DEFAULT_ADJ_TABLE = Hash.new { |h, k| h[k] = Hash.new(&h.default_proc) }
   [
-    ['A', 'E'], ['A', 'I'], ['A', 'O'], ['A', 'U'], ['B', 'V'], ['E', 'I'], ['E', 'O'], ['E', 'U'], ['I', 'O'],
-    ['I', 'U'], ['O', 'U'], ['I', 'Y'], ['E', 'Y'], ['C', 'G'], ['E', 'F'], ['W', 'U'], ['W', 'V'], ['X', 'K'],
-    ['S', 'Z'], ['X', 'S'], ['Q', 'C'], ['U', 'V'], ['M', 'N'], ['L', 'I'], ['Q', 'O'], ['P', 'R'], ['I', 'J'],
-    ['2', 'Z'], ['5', 'S'], ['8', 'B'], ['1', 'I'], ['1', 'L'], ['0', 'O'], ['0', 'Q'], ['C', 'K'], ['G', 'J'],
+    %w[A E], %w[A I], %w[A O], %w[A U], %w[B V], %w[E I], %w[E O], %w[E U], %w[I O],
+    %w[I U], %w[O U], %w[I Y], %w[E Y], %w[C G], %w[E F], %w[W U], %w[W V], %w[X K],
+    %w[S Z], %w[X S], %w[Q C], %w[U V], %w[M N], %w[L I], %w[Q O], %w[P R], %w[I J],
+    %w[2 Z], %w[5 S], %w[8 B], %w[1 I], %w[1 L], %w[0 O], %w[0 Q], %w[C K], %w[G J],
     ['E', ' '], ['Y', ' '], ['S', ' ']
-  ].each{ |s1, s2|
-    if not DEFAULT_ADJ_TABLE.has_key?(s1)
-      DEFAULT_ADJ_TABLE[s1] = Hash.new
-    end
-    if not DEFAULT_ADJ_TABLE.has_key?(s2)
-      DEFAULT_ADJ_TABLE[s2] = Hash.new
-    end
+  ].each do |s1, s2|
     DEFAULT_ADJ_TABLE[s1][s2] = DEFAULT_ADJ_TABLE[s2][s1] = true
-  }
-  DEFAULT_ADJ_TABLE.default = Hash.new
+  end
 end

data/lib/jaro_winkler/jaro_winkler_pure.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'jaro_winkler/adjusting_table'
 module JaroWinkler
   class Error < RuntimeError; end
@@ -6,120 +8,122 @@ module JaroWinkler
   DEFAULT_WEIGHT = 0.1
   DEFAULT_THRESHOLD = 0.7
   DEFAULT_OPTIONS = {
-    jaro: {adj_table: false, ignore_case: false},
-    jaro_winkler: {weight: DEFAULT_WEIGHT, threshold: DEFAULT_THRESHOLD}
-  }
-
-  module_function
-
-  def distance str1, str2, options={}
-    _distance str1.codepoints.to_a, str2.codepoints.to_a, options
-  end
+    jaro: { adj_table: false, ignore_case: false },
+    jaro_winkler: { weight: DEFAULT_WEIGHT, threshold: DEFAULT_THRESHOLD }
+  }.freeze
 
-  def jaro_distance str1, str2, options={}
-    _jaro_distance str1.codepoints.to_a, str2.codepoints.to_a, options
-  end
+  class << self
+    def distance(str1, str2, options = {})
+      _distance str1.codepoints.to_a, str2.codepoints.to_a, options
+    end
 
-  def _distance codes1, codes2, options={}
-    options = DEFAULT_OPTIONS[:jaro_winkler].merge options
-    raise InvalidWeightError if options[:weight] > 0.25
-    jaro_distance = _jaro_distance(codes1, codes2, options);
+    def jaro_distance(str1, str2, options = {})
+      _jaro_distance str1.codepoints.to_a, str2.codepoints.to_a, options
+    end
 
-    if jaro_distance < options[:threshold]
-      jaro_distance
-    else
-      codes1, codes2 = codes2, codes1 if codes1.length > codes2.length
-      len1, len2 = codes1.length, codes2.length
-      max_4 = len1 > 4 ? 4 : len1
-      prefix = 0
-      while prefix < max_4 && codes1[prefix] == codes2[prefix]
-        prefix += 1
+    private
+
+    def _distance(codes1, codes2, options = {})
+      options = DEFAULT_OPTIONS[:jaro_winkler].merge options
+      raise InvalidWeightError if options[:weight] > 0.25
+      jaro_distance = _jaro_distance(codes1, codes2, options)
+
+      if jaro_distance < options[:threshold]
+        jaro_distance
+      else
+        codes1, codes2 = codes2, codes1 if codes1.length > codes2.length
+        len1 = codes1.length
+        len2 = codes2.length
+        max_4 = len1 > 4 ? 4 : len1
+        prefix = 0
+        prefix += 1 while prefix < max_4 && codes1[prefix] == codes2[prefix]
+        jaro_distance + prefix * options[:weight] * (1 - jaro_distance)
       end
-      jaro_distance + prefix * options[:weight] * (1 - jaro_distance)
     end
-  end
-
-  def _jaro_distance codes1, codes2, options={}
-    options = DEFAULT_OPTIONS[:jaro].merge options
 
-    codes1, codes2 = codes2, codes1 if codes1.length > codes2.length
-    len1, len2 = codes1.length, codes2.length
-    return 0.0 if len1 == 0 || len2 == 0
+    def _jaro_distance(codes1, codes2, options = {})
+      options = DEFAULT_OPTIONS[:jaro].merge options
 
-    if options[:ignore_case]
-      codes1.map!{ |c| c >= 97 && c <= 122 ? c -= 32 : c }
-      codes2.map!{ |c| c >= 97 && c <= 122 ? c -= 32 : c }
-    end
+      codes1, codes2 = codes2, codes1 if codes1.length > codes2.length
+      len1 = codes1.length
+      len2 = codes2.length
+      return 0.0 if len1 == 0 || len2 == 0
 
-    window = len2/2 - 1
-    window = 0 if(window < 0)
-    flags1, flags2 = 0, 0
-
-    # // count number of matching characters
-    match_count = 0;
-    i = 0
-    while i < len1
-      left = (i >= window) ? i - window : 0
-      right = (i + window <= len2 - 1) ? (i + window) : (len2 - 1)
-      right = len2 - 1 if right > len2 - 1
-      j = left
-      while j <= right
-        if flags2[j] == 0 && codes1[i] == codes2[j]
-          flags1 |= (1 << i)
-          flags2 |= (1 << j)
-          match_count += 1
-          break
-        end
-        j +=1
+      if options[:ignore_case]
+        codes1.map! { |c| c >= 97 && c <= 122 ? c -= 32 : c }
+        codes2.map! { |c| c >= 97 && c <= 122 ? c -= 32 : c }
       end
-      i += 1
-    end
 
-    return 0.0 if match_count == 0
-
-    # // count number of transpositions
-    transposition_count = j = k = 0
-    i = 0
-    while i < len1
-      if flags1[i] == 1
-        j = k
-        while j < len2
-          if flags2[j] == 1
-            k = j + 1;
-            break;
+      window = len2 / 2 - 1
+      window = 0 if window < 0
+      flags1 = 0
+      flags2 = 0
+
+      # // count number of matching characters
+      match_count = 0
+      i = 0
+      while i < len1
+        left = i >= window ? i - window : 0
+        right = i + window <= len2 - 1 ? (i + window) : (len2 - 1)
+        right = len2 - 1 if right > len2 - 1
+        j = left
+        while j <= right
+          if flags2[j] == 0 && codes1[i] == codes2[j]
+            flags1 |= (1 << i)
+            flags2 |= (1 << j)
+            match_count += 1
+            break
           end
           j += 1
         end
-        transposition_count += 1 if codes1[i] != codes2[j]
+        i += 1
       end
-      i += 1
-    end
 
-    # // count similarities in nonmatched characters
-    similar_count = 0
-    if options[:adj_table] && len1 > match_count
+      return 0.0 if match_count == 0
+
+      # // count number of transpositions
+      transposition_count = j = k = 0
       i = 0
       while i < len1
-        if flags1[i] == 0
-          j = 0
+        if flags1[i] == 1
+          j = k
           while j < len2
-            if flags2[j] == 0
-              if DEFAULT_ADJ_TABLE[codes1[i].chr(Encoding::UTF_8)][codes2[j].chr(Encoding::UTF_8)]
-                similar_count += 3
-                break
-              end
+            if flags2[j] == 1
+              k = j + 1
+              break
             end
             j += 1
           end
+          transposition_count += 1 if codes1[i] != codes2[j]
         end
         i += 1
       end
-    end
 
-    m = match_count.to_f
-    t = transposition_count/2
-    m = similar_count/10.0 + m if options[:adj_table]
-    (m/len1 + m/len2 + (m-t)/m) / 3
-  end
+      # // count similarities in nonmatched characters
+      similar_count = 0
+      if options[:adj_table] && len1 > match_count
+        i = 0
+        while i < len1
+          if flags1[i] == 0
+            j = 0
+            while j < len2
+              if flags2[j] == 0
+                if DEFAULT_ADJ_TABLE[codes1[i].chr(Encoding::UTF_8)][codes2[j].chr(Encoding::UTF_8)]
+                  similar_count += 3
+                  break
+                end
+              end
+              j += 1
+            end
+          end
+          i += 1
+        end
+      end
 
-end
+      m = match_count.to_f
+      t = transposition_count / 2
+      m = similar_count / 10.0 + m if options[:adj_table]
+      (m / len1 + m / len2 + (m - t) / m) / 3
+    end
+  end
+end

data/lib/jaro_winkler/version.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module JaroWinkler
-  VERSION = '1.4.0'
+  VERSION = '1.5.0'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: jaro_winkler
 version: !ruby/object:Gem::Version
-  version: 1.4.0
+  version: 1.5.0
 platform: ruby
 authors:
 - Jian Weihang
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-12-12 00:00:00.000000000 Z
+date: 2017-10-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -66,9 +66,12 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-description: It's a implementation of Jaro-Winkler distance algorithm, it uses C extension
-  and will fallback to pure Ruby version in JRuby. Both implementation supports UTF-8
-  string.
+description: |-
+  jaro_winkler is an implementation of Jaro-Winkler \
+    distance algorithm which is written in C extension and will fallback to pure \
+    Ruby version in platforms other than MRI/KRI like JRuby or Rubinius. Both of \
+    C and Ruby implementation support any kind of string encoding, such as \
+    UTF-8, EUC-JP, Big5, etc.
 email: tonytonyjan@gmail.com
 executables: []
 extensions:
@@ -77,13 +80,12 @@ extra_rdoc_files: []
 files:
 - ext/jaro_winkler/adj_matrix.c
 - ext/jaro_winkler/adj_matrix.h
-- ext/jaro_winkler/code.c
-- ext/jaro_winkler/code.h
+- ext/jaro_winkler/codepoints.c
+- ext/jaro_winkler/codepoints.h
 - ext/jaro_winkler/extconf.rb
 - ext/jaro_winkler/jaro.c
 - ext/jaro_winkler/jaro.h
 - ext/jaro_winkler/jaro_winkler.c
-- ext/jaro_winkler/murmur_hash2.c
 - lib/jaro_winkler.rb
 - lib/jaro_winkler/adjusting_table.rb
 - lib/jaro_winkler/jaro_winkler_pure.rb
@@ -108,9 +110,9 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.5.1
+rubygems_version: 2.6.12
 signing_key: 
 specification_version: 4
-summary: Ruby & C implementation of Jaro-Winkler distance algorithm which both support
-  UTF-8 string.
+summary: An implementation of Jaro-Winkler distance algorithm written \ in C extension
+  which supports any kind of string encoding.
 test_files: []

data/ext/jaro_winkler/code.c

@@ -1,29 +0,0 @@
-#include <stdlib.h>
-#include <string.h>
-
-void utf_char_to_code(char *str, unsigned long long *ret_code, int *ret_byte_length){
-  unsigned char first_char = str[0];
-  if(first_char >= 252) *ret_byte_length = 6;      // 1111110x
-  else if(first_char >= 248) *ret_byte_length = 5; // 111110xx
-  else if(first_char >= 240) *ret_byte_length = 4; // 11110xxx
-  else if(first_char >= 224) *ret_byte_length = 3; // 1110xxxx
-  else if(first_char >= 192) *ret_byte_length = 2; // 110xxxxx
-  else *ret_byte_length = 1;
-  *ret_code = 0;
-  memcpy(ret_code, str, *ret_byte_length);
-}
-
-void string_to_codes(char *str, int length, unsigned long long **ret_codes, int *ret_length){
-  unsigned int code;
-  char byte_length;
-
-  *ret_codes = calloc(length, sizeof(long long));
-  *ret_length = 0;
-
-  for(int i = 0; i < length;){
-    int byte_length;
-    utf_char_to_code(&str[i], &(*ret_codes)[*ret_length], &byte_length);
-    *ret_length += 1;
-    i += byte_length;
-  }
-}

data/ext/jaro_winkler/code.h

@@ -1,7 +0,0 @@
-#ifndef CODE_H
-#define CODE_H
-
-void utf_char_to_code(char *str, unsigned long long *ret_code, int *ret_byte_length);
-void string_to_codes(char *str, int length, unsigned long long **ret_codes, int *ret_length);
-
-#endif

data/ext/jaro_winkler/murmur_hash2.c

@@ -1,64 +0,0 @@
-//-----------------------------------------------------------------------------
-// MurmurHash2, by Austin Appleby
-
-// Note - This code makes a few assumptions about how your machine behaves -
-
-// 1. We can read a 4-byte value from any address without crashing
-// 2. sizeof(int) == 4
-
-// And it has a few limitations -
-
-// 1. It will not work incrementally.
-// 2. It will not produce the same results on little-endian and big-endian
-//    machines.
-
-unsigned int MurmurHash2 ( const void * key, int len, unsigned int seed )
-{
-  // 'm' and 'r' are mixing constants generated offline.
-  // They're not really 'magic', they just happen to work well.
-
-  const unsigned int m = 0x5bd1e995;
-  const int r = 24;
-
-  // Initialize the hash to a 'random' value
-
-  unsigned int h = seed ^ len;
-
-  // Mix 4 bytes at a time into the hash
-
-  const unsigned char * data = (const unsigned char *)key;
-
-  while(len >= 4)
-  {
-    unsigned int k = *(unsigned int *)data;
-
-    k *= m;
-    k ^= k >> r;
-    k *= m;
-
-    h *= m;
-    h ^= k;
-
-    data += 4;
-    len -= 4;
-  }
-  
-  // Handle the last few bytes of the input array
-
-  switch(len)
-  {
-  case 3: h ^= data[2] << 16;
-  case 2: h ^= data[1] << 8;
-  case 1: h ^= data[0];
-          h *= m;
-  };
-
-  // Do a few final mixes of the hash to ensure the last few
-  // bytes are well-incorporated.
-
-  h ^= h >> 13;
-  h *= m;
-  h ^= h >> 15;
-
-  return h;
-}