str_metrics 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: f773ff65c7ef2fb3bc5f396cf10e13e7d98d3037981de45a846f8b9fc6328caa
+  data.tar.gz: c6b3838b89a063c4e57cf70e06a7e9226bee9ecb84a99d0f2773775dbb56cc6e
+SHA512:
+  metadata.gz: 22df7813421a8850d277065d8595aadf5a30c0fc96ebfb90bfaac05e3906030c5874dd4a8da38e1a0af67277f22ca2c46940515d509ed6bd63c0d332e6620b3b
+  data.tar.gz: 7addd087a16eb72ed3428a9b90ce51c0e0ed5865bc2d8aebe7b899f2adc5fdbbcad15c73b60af3701213129e9f7fed6e69a834637f7e32c14f478f8caaa7e589

data/Cargo.toml

@@ -0,0 +1,15 @@
+[package]
+name = "str_metrics"
+version = "0.1.0"
+authors = ["Anirban Mukhopadhyay <anirban.mukhop@gmail.com>"]
+edition = "2018"
+
+[lib]
+crate-type = ["cdylib"]
+
+[dependencies]
+unicode-segmentation = "^1.6"
+libc = "^0.2"
+itertools = "^0.8"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html

data/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2020 Anirban Mukhopadhyay
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

data/README.md

@@ -0,0 +1,170 @@
+# StrMetrics
+
+[![checks](https://github.com/anirbanmu/str_metrics/workflows/checks/badge.svg)](https://github.com/anirbanmu/str_metrics/actions?query=workflow%3Achecks)
+
+Ruby gem (native extension in Rust) providing implementations of various string metrics. Current metrics supported are: Sørensen–Dice, Levenshtein, Damerau–Levenshtein, Jaro & Jaro–Winkler. Strings that are UTF-8 encodable (convertible to UTF-8 representation) are supported. All comparison of strings is done at the grapheme cluster level as described by [Unicode Standard Annex #29](https://www.unicode.org/reports/tr29/#Grapheme_Cluster_Boundaries); this may be different from many gems that calculate string metrics.
+
+## Getting Started
+### Prerequisites
+
+Install Rust (tested with version `>= 1.38.0`) with:
+
+```sh
+curl https://sh.rustup.rs -sSf | sh
+```
+
+### Installation
+
+#### With [`bundler`](https://bundler.io/)
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'str_metrics'
+```
+
+And then execute:
+
+    $ bundle install
+
+#### Without `bundler`
+
+    $ gem install str_metrics
+
+## Usage
+
+All you need to do to use the metrics provided in this gem is to make sure `str_metrics` is required like:
+
+```ruby
+require 'str_metrics'
+```
+
+Each metric is shown below with an example & meanings of optional parameters.
+
+### Sørensen–Dice
+
+```ruby
+StrMetrics::SorensenDice.coefficient('abc', 'bcd', ignore_case: false)
+ => 0.5
+```
+Options:
+
+Keyword | Type | Default | Description
+--- | --- | --- | ---
+`ignore_case` | boolean | `false` | Case insensitive comparison?
+
+### Levenshtein
+
+```ruby
+StrMetrics::Levenshtein.distance('abc', 'acb', ignore_case: false)
+ => 2
+```
+Options:
+
+Keyword | Type | Default | Description
+--- | --- | --- | ---
+`ignore_case` | boolean | `false` | Case insensitive comparison?
+
+### Damerau–Levenshtein
+
+```ruby
+StrMetrics::DamerauLevenshtein.distance('abc', 'acb', ignore_case: false)
+ => 1
+```
+Options:
+
+Keyword | Type | Default | Description
+--- | --- | --- | ---
+`ignore_case` | boolean | `false` | Case insensitive comparison?
+
+### Jaro
+
+```ruby
+StrMetrics::Jaro.similarity('abc', 'aac', ignore_case: false)
+ => 0.7777777777777777
+```
+Options:
+
+Keyword | Type | Default | Description
+--- | --- | --- | ---
+`ignore_case` | boolean | `false` | Case insensitive comparison?
+
+### Jaro–Winkler
+
+```ruby
+StrMetrics::JaroWinkler.similarity('abc', 'aac', ignore_case: false, prefix_scaling_factor: 0.1, prefix_scaling_bonus_threshold: 0.7)
+ => 0.7999999999999999
+
+StrMetrics::JaroWinkler.distance('abc', 'aac', ignore_case: false, prefix_scaling_factor: 0.1, prefix_scaling_bonus_threshold: 0.7)
+ => 0.20000000000000007
+```
+Options:
+
+Keyword | Type | Default | Description
+--- | --- | --- | ---
+`ignore_case` | boolean | `false` | Case insensitive comparison?
+`prefix_scaling_factor` | decimal | `0.1` | Constant scaling factor for how much to weight common prefixes. Should not exceed 0.25.
+`prefix_scaling_bonus_threshold` | decimal | `0.7` | Prefix bonus weighting will only be applied if the Jaro similarity is greater given value.
+
+## Motivation
+
+The main motivation was to have a central gem which can provide a variety of string metric calculations. Secondary motivation was to experiment with writing a native extension in Rust (instead of C).
+
+## Development
+
+### Getting started
+
+```bash
+gem install bundler
+git clone https://github.com/anirbanmu/str_metrics.git
+cd ./str_metrics
+bundle install
+```
+
+### Building (for native component)
+
+```bash
+rake rust_build
+```
+
+### Testing (will build native component before running tests)
+```bash
+rake spec
+```
+
+### Local installation
+```bash
+rake install
+```
+
+### Deploying a new version
+To deploy a new version of the gem to rubygems:
+
+1. Bump version in [version.rb](lib/str_metrics/version.rb) according to [SemVer](https://semver.org/).
+2. Get your code merged to master
+3. After a `git pull` on master:
+
+```bash
+rake build && rake release
+```
+
+## Authors
+- [Anirban Mukhopadhyay](https://github.com/anirbanmu)
+
+See all repo contributors [here](https://github.com/anirbanmu/str_metrics/contributors).
+
+## Versioning
+
+[SemVer](https://semver.org/) is employed. See [tags](https://github.com/anirbanmu/str_metrics/tags) for released versions.
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/anirbanmu/str_metrics.
+
+## Code of Conduct
+
+Everyone interacting in this project's codebase, issue trackers etc. are expected to follow the [code of conduct](CODE_OF_CONDUCT.md).
+
+## License
+
+This project is licensed under the MIT License - see the [LICENSE](LICENSE) file for details

data/extconf.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+abort 'Rust compiler required (https://www.rust-lang.org/)' if `which rustc`.empty?
+
+File.open('Makefile', 'wb') do |f|
+  f.puts(<<~MKCONTENT)
+    all:
+    \tcargo rustc --release
+    \tmv ./target/release/libstr_metrics.so ./lib/str_metrics
+    clean:
+    install:
+    \trm -r target
+  MKCONTENT
+end

data/lib/str_metrics/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module StrMetrics
+  VERSION = '0.1.0'
+end

data/lib/str_metrics.rb

@@ -0,0 +1,96 @@
+# frozen_string_literal: true
+
+require 'ffi'
+require 'str_metrics/version'
+
+# Namespace for gem
+module StrMetrics
+  # Interface with Rust functions (not meant for public usage)
+  module Native
+    extend FFI::Library
+
+    ffi_lib File.expand_path('./str_metrics/libstr_metrics.so', __dir__)
+
+    attach_function :sorensen_dice_coefficient, %i[string string char], :double
+    attach_function :levenshtein_distance, %i[string string char], :int64
+    attach_function :damerau_levenshtein_distance, %i[string string char], :int64
+    attach_function :jaro_similarity, %i[string string char], :double
+    attach_function :jaro_winkler_similarity, %i[string string char int double double], :double
+    attach_function :jaro_winkler_distance, %i[string string char int double double], :double
+  end
+
+  private_constant :Native
+
+  refine String do
+    def to_utf8
+      encoding == Encoding::UTF_8 ? self : encode('UTF-8')
+    end
+  end
+
+  using self # activates refinement
+
+  # Namespace for Sorensen-Dice
+  module SorensenDice
+    def self.coefficient(a, b, ignore_case: false)
+      Native.sorensen_dice_coefficient(a&.to_utf8, b&.to_utf8, ignore_case ? 1 : 0)
+    end
+  end
+
+  # Namespace for Levenshtein
+  module Levenshtein
+    def self.distance(a, b, ignore_case: false)
+      Native.levenshtein_distance(a&.to_utf8, b&.to_utf8, ignore_case ? 1 : 0)
+    end
+  end
+
+  # Namespace for Damerau-Levenshtein
+  module DamerauLevenshtein
+    def self.distance(a, b, ignore_case: false)
+      Native.damerau_levenshtein_distance(a&.to_utf8, b&.to_utf8, ignore_case ? 1 : 0)
+    end
+  end
+
+  # Namespace for Jaro
+  module Jaro
+    def self.similarity(a, b, ignore_case: false)
+      Native.jaro_similarity(a&.to_utf8, b&.to_utf8, ignore_case ? 1 : 0)
+    end
+  end
+
+  # Namespace for Jaro-Winkler
+  module JaroWinkler
+    def self.similarity(
+      a,
+      b,
+      ignore_case: false,
+      prefix_scaling_factor: 0.1,
+      prefix_scaling_bonus_threshold: 0.7
+    )
+      Native.jaro_winkler_similarity(
+        a&.to_utf8,
+        b&.to_utf8,
+        ignore_case ? 1 : 0,
+        4,
+        prefix_scaling_factor,
+        prefix_scaling_bonus_threshold
+      )
+    end
+
+    def self.distance(
+      a,
+      b,
+      ignore_case: false,
+      prefix_scaling_factor: 0.1,
+      prefix_scaling_bonus_threshold: 0.7
+    )
+      Native.jaro_winkler_distance(
+        a&.to_utf8,
+        b&.to_utf8,
+        ignore_case ? 1 : 0,
+        4,
+        prefix_scaling_factor,
+        prefix_scaling_bonus_threshold
+      )
+    end
+  end
+end

data/src/lib.rs

@@ -0,0 +1,167 @@
+mod metrics;
+
+use libc::{c_char, c_double};
+use std::ffi::CStr;
+
+fn cstr_from_raw(s: &*const c_char) -> &CStr {
+    unsafe { CStr::from_ptr(*s) }
+}
+
+#[no_mangle]
+pub extern "C" fn sorensen_dice_coefficient(
+    a: *const c_char,
+    b: *const c_char,
+    ignore_case: c_char,
+) -> c_double {
+    if a.is_null() || b.is_null() {
+        return 0.0;
+    }
+
+    let a_c_str = cstr_from_raw(&a);
+    let b_c_str = cstr_from_raw(&b);
+
+    let a_str = match a_c_str.to_str() {
+        Err(_e) => return 0.0,
+        Ok(s) => s,
+    };
+
+    let b_str = match b_c_str.to_str() {
+        Err(_e) => return 0.0,
+        Ok(s) => s,
+    };
+
+    metrics::sorensen_dice::coefficient(a_str, b_str, ignore_case == 1)
+}
+
+#[no_mangle]
+pub extern "C" fn jaro_similarity(
+    a: *const c_char,
+    b: *const c_char,
+    ignore_case: c_char,
+) -> c_double {
+    if a.is_null() || b.is_null() {
+        return 0.0;
+    }
+
+    let a_c_str = cstr_from_raw(&a);
+    let b_c_str = cstr_from_raw(&b);
+
+    let a_str = match a_c_str.to_str() {
+        Err(_e) => return 0.0,
+        Ok(s) => s,
+    };
+
+    let b_str = match b_c_str.to_str() {
+        Err(_e) => return 0.0,
+        Ok(s) => s,
+    };
+
+    metrics::jaro::similarity(a_str, b_str, ignore_case == 1).value
+}
+
+#[no_mangle]
+pub extern "C" fn jaro_winkler_similarity(
+    a: *const c_char,
+    b: *const c_char,
+    ignore_case: c_char,
+    prefix_length: u32,
+    prefix_scaling_factor: c_double,
+    prefix_scaling_bonus_threshold: c_double,
+) -> c_double {
+    if a.is_null() || b.is_null() {
+        return 0.0;
+    }
+
+    let a_c_str = cstr_from_raw(&a);
+    let b_c_str = cstr_from_raw(&b);
+
+    let a_str = match a_c_str.to_str() {
+        Err(_e) => return 0.0,
+        Ok(s) => s,
+    };
+
+    let b_str = match b_c_str.to_str() {
+        Err(_e) => return 0.0,
+        Ok(s) => s,
+    };
+
+    metrics::jaro_winkler::similarity(
+        a_str,
+        b_str,
+        ignore_case == 1,
+        prefix_length,
+        prefix_scaling_factor,
+        prefix_scaling_bonus_threshold,
+    )
+}
+
+#[no_mangle]
+pub extern "C" fn jaro_winkler_distance(
+    a: *const c_char,
+    b: *const c_char,
+    ignore_case: c_char,
+    prefix_length: u32,
+    prefix_scaling_factor: c_double,
+    prefix_scaling_bonus_threshold: c_double,
+) -> c_double {
+    1.0 - jaro_winkler_similarity(
+        a,
+        b,
+        ignore_case,
+        prefix_length,
+        prefix_scaling_factor,
+        prefix_scaling_bonus_threshold,
+    )
+}
+
+#[no_mangle]
+pub extern "C" fn levenshtein_distance(
+    a: *const c_char,
+    b: *const c_char,
+    ignore_case: c_char,
+) -> i64 {
+    if a.is_null() || b.is_null() {
+        return std::i64::MAX;
+    }
+
+    let a_c_str = cstr_from_raw(&a);
+    let b_c_str = cstr_from_raw(&b);
+
+    let a_str = match a_c_str.to_str() {
+        Err(_e) => return std::i64::MAX,
+        Ok(s) => s,
+    };
+
+    let b_str = match b_c_str.to_str() {
+        Err(_e) => return std::i64::MAX,
+        Ok(s) => s,
+    };
+
+    metrics::levenshtein::distance(a_str, b_str, ignore_case == 1)
+}
+
+#[no_mangle]
+pub extern "C" fn damerau_levenshtein_distance(
+    a: *const c_char,
+    b: *const c_char,
+    ignore_case: c_char,
+) -> i64 {
+    if a.is_null() || b.is_null() {
+        return std::i64::MAX;
+    }
+
+    let a_c_str = cstr_from_raw(&a);
+    let b_c_str = cstr_from_raw(&b);
+
+    let a_str = match a_c_str.to_str() {
+        Err(_e) => return std::i64::MAX,
+        Ok(s) => s,
+    };
+
+    let b_str = match b_c_str.to_str() {
+        Err(_e) => return std::i64::MAX,
+        Ok(s) => s,
+    };
+
+    metrics::damerau_levenshtein::distance(a_str, b_str, ignore_case == 1)
+}

data/src/metrics/damerau_levenshtein.rs

@@ -0,0 +1,71 @@
+use crate::metrics::utils::graphemes;
+use crate::metrics::utils::Array2D;
+
+use std::cmp;
+use std::collections::HashMap;
+use std::hash::Hash;
+
+pub fn distance(a: &str, b: &str, ignore_case: bool) -> i64 {
+    if ignore_case {
+        return distance_impl(&graphemes(&a.to_lowercase()), &graphemes(&b.to_lowercase()));
+    }
+
+    distance_impl(&graphemes(a), &graphemes(b))
+}
+
+// From https://en.wikipedia.org/wiki/Damerau-Levenshtein_distance
+fn distance_impl<T: Hash + Eq>(a: &[T], b: &[T]) -> i64 {
+    let lens = [a.len(), b.len()];
+    if lens[0] == 0 {
+        return lens[1] as i64;
+    }
+    if lens[1] == 0 {
+        return lens[0] as i64;
+    }
+
+    let rows = lens[0] + 2;
+    let columns = lens[1] + 2;
+
+    let max_dist = (lens[0] + lens[1]) as i64;
+    let mut dist_matrix = Array2D::new(rows, columns);
+
+    dist_matrix[(0, 0)] = max_dist;
+    for i in 1..rows {
+        dist_matrix[(i, 0)] = max_dist;
+        dist_matrix[(i, 1)] = (i - 1) as i64;
+    }
+    for j in 1..columns {
+        dist_matrix[(0, j)] = max_dist;
+        dist_matrix[(1, j)] = (j - 1) as i64;
+    }
+
+    let mut da: HashMap<&T, usize> = HashMap::new();
+
+    for i in 1..=lens[0] {
+        let mut db = 0;
+        for j in 1..=lens[1] {
+            let k = da.entry(&b[j - 1]).or_insert(0);
+            let l = db;
+            let cost = if a[i - 1] == b[j - 1] {
+                db = j;
+                0
+            } else {
+                1
+            };
+
+            dist_matrix[(i + 1, j + 1)] = cmp::min(
+                dist_matrix[(i, j)] + cost,
+                cmp::min(
+                    dist_matrix[(i + 1, j)] + 1,
+                    cmp::min(
+                        dist_matrix[(i, j + 1)] + 1,
+                        dist_matrix[(*k, l)] + (i - *k - 1) as i64 + 1 + (j - l - 1) as i64,
+                    ),
+                ),
+            );
+        }
+        da.insert(&a[i - 1], i);
+    }
+
+    dist_matrix[(lens[0] + 1, lens[1] + 1)]
+}

data/src/metrics/jaro.rs

@@ -0,0 +1,93 @@
+use crate::metrics::utils::graphemes;
+use std::cmp;
+
+pub struct JaroSimilarityResult {
+    pub value: f64,
+    pub max_prefix_length: i64,
+}
+
+pub fn similarity(a: &str, b: &str, ignore_case: bool) -> JaroSimilarityResult {
+    if ignore_case {
+        return similarity_impl(&graphemes(&a.to_lowercase()), &graphemes(&b.to_lowercase()));
+    }
+
+    similarity_impl(&graphemes(a), &graphemes(b))
+}
+
+fn similarity_impl<T: Eq>(a: &[T], b: &[T]) -> JaroSimilarityResult {
+    let mut graphemes = [a, b];
+    if graphemes[0].len() > graphemes[1].len() {
+        graphemes.swap(0, 1);
+    }
+
+    // let grapheme_iterators = [UnicodeSegmentation::graphemes(&case_handled[0][..], true), UnicodeSegmentation::graphemes(&case_handled[1][..], true)];
+    let lens = [graphemes[0].len(), graphemes[1].len()];
+
+    let max_length = cmp::max(lens[0], lens[1]);
+    let matching_dist = if max_length < 2 {
+        0
+    } else {
+        (max_length / 2) - 1
+    };
+
+    let mut matching_indices = [
+        Vec::with_capacity(max_length),
+        Vec::with_capacity(max_length),
+    ];
+
+    // Find matches
+    let mut last_matched_prefix_index = -1;
+    {
+        let mut b_matched = vec![false; lens[1]];
+        for (i, grapheme) in graphemes[0].iter().enumerate() {
+            let start = cmp::max(0 as i64, i as i64 - matching_dist as i64) as usize;
+            let end = cmp::min(lens[1], i + matching_dist + 1);
+
+            // Keep track of prefix match
+            // Safe to access i in b since a.len < b.len
+            if grapheme == &graphemes[1][i]
+                && ((last_matched_prefix_index == -1 && i == 0)
+                    || last_matched_prefix_index == i as i64 - 1)
+            {
+                last_matched_prefix_index = i as i64;
+            }
+
+            for (j, matched) in b_matched.iter_mut().enumerate().take(end).skip(start) {
+                if grapheme == &graphemes[1][j] && !*matched {
+                    *matched = true;
+                    matching_indices[0].push(i);
+                    matching_indices[1].push(j);
+                    break;
+                }
+            }
+        }
+    }
+
+    let matches = matching_indices[0].len();
+    if matches == 0 {
+        return JaroSimilarityResult {
+            value: 0.0,
+            max_prefix_length: 0,
+        };
+    }
+
+    // Find transpositions / 2 in matches
+    matching_indices[1].sort_unstable();
+    let transpositions = matching_indices[0]
+        .iter()
+        .zip(matching_indices[1].iter())
+        .fold(0.0, |acc, (i, j)| {
+            if graphemes[0][*i] == graphemes[1][*j] {
+                acc
+            } else {
+                acc + 0.5
+            }
+        });
+
+    let m = matches as f64;
+    let t = transpositions;
+    JaroSimilarityResult {
+        value: ((m / lens[0] as f64) + (m / lens[1] as f64) + ((m - t) / m)) / 3.0,
+        max_prefix_length: last_matched_prefix_index + 1,
+    }
+}

data/src/metrics/jaro_winkler.rs

@@ -0,0 +1,20 @@
+use crate::metrics::jaro;
+use std::cmp;
+
+pub fn similarity(
+    a: &str,
+    b: &str,
+    ignore_case: bool,
+    prefix_length: u32,
+    prefix_scaling_factor: f64,
+    prefix_scaling_bonus_threshold: f64,
+) -> f64 {
+    let jaro_similarity = jaro::similarity(a, b, ignore_case);
+    let common_prefix_len = cmp::min(prefix_length as i64, jaro_similarity.max_prefix_length);
+
+    if jaro_similarity.value > prefix_scaling_bonus_threshold {
+        return jaro_similarity.value
+            + common_prefix_len as f64 * prefix_scaling_factor * (1.0 - jaro_similarity.value);
+    }
+    jaro_similarity.value
+}

data/src/metrics/levenshtein.rs

@@ -0,0 +1,46 @@
+use crate::metrics::utils::graphemes;
+use crate::metrics::utils::Array2D;
+use std::cmp;
+
+pub fn distance(a: &str, b: &str, ignore_case: bool) -> i64 {
+    if ignore_case {
+        return distance_impl(&graphemes(&a.to_lowercase()), &graphemes(&b.to_lowercase()));
+    }
+
+    distance_impl(&graphemes(a), &graphemes(b))
+}
+
+fn distance_impl<T: Eq>(a: &[T], b: &[T]) -> i64 {
+    let lens = [a.len(), b.len()];
+    if lens[0] == 0 {
+        return lens[1] as i64;
+    }
+    if lens[1] == 0 {
+        return lens[0] as i64;
+    }
+
+    let rows = lens[0] + 1;
+    let columns = lens[1] + 1;
+
+    let mut dist_matrix = Array2D::new(rows, columns);
+
+    for i in 0..rows {
+        dist_matrix[(i, 0)] = i as i64;
+    }
+    for j in 0..columns {
+        dist_matrix[(0, j)] = j as i64;
+    }
+
+    for i in 1..rows {
+        for j in 1..columns {
+            let cost = if a[i - 1] == b[j - 1] { 0 } else { 1 };
+
+            dist_matrix[(i, j)] = cmp::min(
+                cmp::min(dist_matrix[(i - 1, j)] + 1, dist_matrix[(i, j - 1)] + 1),
+                dist_matrix[(i - 1, j - 1)] + cost,
+            );
+        }
+    }
+
+    dist_matrix[(rows - 1, columns - 1)]
+}

data/src/metrics/mod.rs

@@ -0,0 +1,6 @@
+pub mod damerau_levenshtein;
+pub mod jaro;
+pub mod jaro_winkler;
+pub mod levenshtein;
+pub mod sorensen_dice;
+mod utils;

data/src/metrics/sorensen_dice.rs

@@ -0,0 +1,47 @@
+use crate::metrics::utils::generate_bigrams;
+use std::collections::hash_map::Entry::{Occupied, Vacant};
+use std::collections::HashMap;
+
+pub fn coefficient(a: &str, b: &str, ignore_case: bool) -> f64 {
+    if ignore_case {
+        return coefficient_impl(&a.to_lowercase(), &b.to_lowercase());
+    }
+    coefficient_impl(a, b)
+}
+
+fn coefficient_impl(a: &str, b: &str) -> f64 {
+    let a_bigrams = generate_bigrams(&a);
+    let mut b_bigrams_hash: HashMap<&str, i64> = HashMap::new();
+
+    let mut total_bigrams = a_bigrams.len();
+
+    {
+        let b_bigrams = generate_bigrams(&b);
+        for s in &b_bigrams {
+            let counter = b_bigrams_hash.entry(s).or_insert(0);
+            *counter += 1;
+        }
+
+        total_bigrams += b_bigrams.len();
+    }
+
+    if total_bigrams == 0 {
+        return 0.0;
+    }
+
+    let mut intersections = 0;
+    for bigram in &a_bigrams {
+        match b_bigrams_hash.entry(bigram) {
+            Vacant(_) => {}
+            Occupied(entry) => {
+                let counter = entry.get();
+                if counter > &0 {
+                    *entry.into_mut() = entry.get() - 1;
+                    intersections += 1;
+                }
+            }
+        }
+    }
+
+    2.0 * intersections as f64 / total_bigrams as f64
+}

data/src/metrics/utils/array_2d.rs

@@ -0,0 +1,30 @@
+use std::ops::Index;
+use std::ops::IndexMut;
+
+pub struct Array2D<T> {
+    arr: Vec<T>,
+    cols: usize,
+}
+
+impl<T: Clone + Default> Array2D<T> {
+    pub fn new(rows: usize, columns: usize) -> Array2D<T> {
+        Array2D {
+            arr: vec![Default::default(); rows * columns],
+            cols: columns,
+        }
+    }
+}
+
+impl<T> Index<(usize, usize)> for Array2D<T> {
+    type Output = T;
+
+    fn index(&self, (y, x): (usize, usize)) -> &Self::Output {
+        &self.arr[y * self.cols + x]
+    }
+}
+
+impl<T> IndexMut<(usize, usize)> for Array2D<T> {
+    fn index_mut(&mut self, (y, x): (usize, usize)) -> &mut Self::Output {
+        &mut self.arr[y * self.cols + x]
+    }
+}

data/src/metrics/utils/mod.rs

@@ -0,0 +1,19 @@
+extern crate itertools;
+extern crate unicode_segmentation;
+
+use itertools::Itertools;
+use unicode_segmentation::UnicodeSegmentation;
+
+mod array_2d;
+pub use array_2d::Array2D;
+
+pub fn graphemes(s: &str) -> Vec<&str> {
+    UnicodeSegmentation::graphemes(s, true).collect::<Vec<&str>>()
+}
+
+pub fn generate_bigrams(s: &str) -> Vec<&str> {
+    UnicodeSegmentation::grapheme_indices(s, true)
+        .tuple_windows()
+        .map(|(a, b)| &s[a.0..b.0 + b.1.len()])
+        .collect::<Vec<&str>>()
+}

metadata

@@ -0,0 +1,167 @@
+--- !ruby/object:Gem::Specification
+name: str_metrics
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Anirban Mukhopadhyay
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2020-03-13 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: ffi
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '12.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '12.0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rubocop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rubocop-performance
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rubocop-rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: 'Ruby gem (native extension in Rust) providing implementations of various
+  string metrics. Current metrics supported are: Sørensen–Dice, Levenshtein, Damerau–Levenshtein,
+  Jaro & Jaro–Winkler. Strings that are UTF-8 encodable (convertible to UTF-8 representation)
+  are supported. All comparison of strings is done at the grapheme cluster level as
+  described by [Unicode Standard Annex #29](https://www.unicode.org/reports/tr29/#Grapheme_Cluster_Boundaries);
+  this may be different from many gems that calculate string metrics.'
+email:
+- anirban.mukhop@gmail.com
+executables: []
+extensions:
+- extconf.rb
+extra_rdoc_files: []
+files:
+- Cargo.toml
+- LICENSE
+- README.md
+- extconf.rb
+- lib/str_metrics.rb
+- lib/str_metrics/version.rb
+- src/lib.rs
+- src/metrics/damerau_levenshtein.rs
+- src/metrics/jaro.rs
+- src/metrics/jaro_winkler.rs
+- src/metrics/levenshtein.rs
+- src/metrics/mod.rs
+- src/metrics/sorensen_dice.rs
+- src/metrics/utils/array_2d.rs
+- src/metrics/utils/mod.rs
+homepage: https://github.com/anirbanmu/str_metrics
+licenses:
+- MIT
+metadata:
+  allowed_push_host: https://rubygems.org
+  homepage_uri: https://github.com/anirbanmu/str_metrics
+  bug_tracker_uri: https://github.com/anirbanmu/str_metrics/issues
+  source_code_uri: https://github.com/anirbanmu/str_metrics
+  changelog_uri: https://github.com/anirbanmu/str_metrics/blob/v0.1.0/CHANGELOG.md
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 2.3.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.0.6
+signing_key: 
+specification_version: 4
+summary: Ruby gem providing native implementations of various string metrics
+test_files: []