levenshtein_str 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: f32f298f472987c392ffb24891f0acf74e85217d14d9eb43e4f7d38c19a63e73
+  data.tar.gz: 34ccf02e8fb61a6c47094fc082f2d278887aca57ade4d41e9276735896155615
+SHA512:
+  metadata.gz: d7f605c8aa11767b27ac77bd7afa89d9581a2541fb9338a68545b30bceb0c2bbe49bcb197f4d13a8365a7a0c1cade2d32b624b1e5f14f646e10fb30e415e4852
+  data.tar.gz: 2ab3c4a7d014d145091ee005322b8523bc4cfc9c5b9812b0bc6b2570b1942bcb3db9841c61506513edc44322fd3d464b7d06d1840179c578b82a956ad11c04a5

data/Gemfile

@@ -0,0 +1,2 @@
+source "https://rubygems.org"
+gemspec

data/Gemfile.lock

@@ -0,0 +1,19 @@
+PATH
+  remote: .
+  specs:
+    levenshtein_str (0.1.0)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    benchmark-ips (2.8.2)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  benchmark-ips
+  levenshtein_str!
+
+BUNDLED WITH
+   2.1.4

data/README.md

@@ -0,0 +1,60 @@
+# String#levenshtein
+
+A performant Ruby gem for getting the [levenshtein distance](https://en.wikipedia.org/wiki/Levenshtein_distance) between 2 strings, with the leg work done in Crystal.
+
+## Installation:
+
+### System Requirements
+
+- crystal
+- MacOS (untested in Linux)
+
+```ruby
+# Gemfile
+gem "levenshtein_str"
+```
+
+then
+
+`bundle install`
+
+## Usage:
+
+```ruby
+# app.rb
+require 'levenshtein_str'
+
+puts "hello".levenshtein("world")
+```
+
+then
+
+`ruby app.rb # => 4`
+
+## Benchmarks
+
+See benchmark.rb for more code info and full results.
+
+`./benchmark`
+
+- "" and "" Same-ish, sometimes Ruby fractionally quicker (no type conversion overheads)
+- "abd" and "abc" Crystal ~2x quicker
+- "abc" and "abcde" Crystal ~2x quicker
+- "abcdefghi" and "0123456789" Crystal ~2.25x quicker
+- [whole alphabet] vs "012345" Crystal ~2.37x quicker
+
+## Template
+
+Based on [this](https://github.com/johansenja/crystal_gem_template) template for writing Ruby gems in Crystal.
+
+## Contributions && Testing
+
+Testing for now can just be done using Ruby:
+
+`rspec`
+
+Contributions are welcome.
+
+## License
+
+[MIT](https://rem.mit-license.org)

data/Rakefile

@@ -0,0 +1,13 @@
+require "bundler/gem_tasks"
+
+desc "compile the Crystal native extensions"
+task :compile do
+  puts "compiling native extensions"
+  `cd ext/levenshtein_str && shards && make clean && make & cd ../../`
+end
+
+desc "cleaning up compiled binaries"
+task :clean do
+  puts "cleaning up extensions"
+  `cd ext/levenshtein_str && make clean && cd ../../`
+end

data/benchmark

@@ -0,0 +1,156 @@
+#!/usr/bin/env ruby
+$: << File.dirname(__FILE__) + "/lib"
+
+# Aiming to mimic how the Crystal looks fairly closely, with a wrapper method,
+# and then a separate method for the logic
+
+def _levenshtein_rb(first, second)
+  len1, len2 = first.size, second.size
+  return len1 if len2.zero?
+  return len2 if len1.zero?
+
+  matrix = [(0..len1).to_a]
+  (1..len2).each do |j|
+    matrix << [j] + [0] * len1
+  end
+
+  (1..len2).each do |i|
+    (1..len1).each do |j|
+      if first[j - 1] == second[i - 1]
+        matrix[i][j] = matrix[i - 1][j - 1]
+      else
+        matrix[i][j] = [
+          matrix[i - 1][j],
+          matrix[i][j - 1],
+          matrix[i - 1][j - 1],
+        ].min + 1
+      end
+    end
+  end
+  return matrix.last.last
+end
+
+class String
+  def levenshtein_rb(second)
+    _levenshtein_rb(self, second)
+  end
+end
+
+def bm
+  load "Rakefile"
+
+  Rake::Task["compile"].invoke
+
+  require "benchmark/ips"
+  require "levenshtein_str"
+
+  # blank strings
+  Benchmark.ips do |bmark|
+    bmark.report("Blank strings (Ruby)") { 100_000.times { "".levenshtein_rb("") } }
+    bmark.report("Blank string (Crystal)") { 100_000.times { "".levenshtein("") } }
+    bmark.compare!
+  end
+
+  # 1 char diff
+  Benchmark.ips do |bmark|
+    bmark.report("abd vs abc (Ruby)") { 100_000.times { "abd".levenshtein_rb("abc") } }
+    bmark.report("abd vs abc (Crystal)") { 100_000.times { "abd".levenshtein("abc") } }
+    bmark.compare!
+  end
+
+  # short diff
+  Benchmark.ips do |bmark|
+    bmark.report("abc vs abcde (Ruby)") { 100_000.times { "abc".levenshtein_rb("abcde") } }
+    bmark.report("abc vs abcde (Crystal)") { 100_000.times { "abc".levenshtein("abcde") } }
+    bmark.compare!
+  end
+
+  # medium diff
+  Benchmark.ips do |bmark|
+    bmark.report("abcdefghi vs 0123456789 (Ruby)") { 100_000.times { "abcdefghi".levenshtein_rb("0123456789") } }
+    bmark.report("abcdefghi vs 0123456789 (Crystal)") { 100_000.times { "abcdefghi".levenshtein("0123456789") } }
+    bmark.compare!
+  end
+
+  # long diff
+  Benchmark.ips do |bmark|
+    bmark.report("[whole alphabet] vs 012345 (Ruby)") { 100_000.times { "abcdefghijklmnopqrstuvwxyz".levenshtein_rb("012345") } }
+    bmark.report("[whole alphabet] vs 012345 (Crystal)") { 100_000.times { "abcdefghijklmnopqrstuvwxyz".levenshtein("012345") } }
+    bmark.compare!
+  end
+end
+
+begin
+  bm
+ensure
+  Rake::Task["clean"].invoke
+end
+
+# RESULTS
+
+# Warming up --------------------------------------
+# Blank strings (Ruby)     3.000  i/100ms
+# Blank string (Crystal)
+#                          3.000  i/100ms
+# Calculating -------------------------------------
+# Blank strings (Ruby)     48.982  (±16.3%) i/s -    237.000  in   5.004116s
+# Blank string (Crystal)
+#                          47.140  (±14.8%) i/s -    231.000  in   5.030098s
+
+# Comparison:
+# Blank strings (Ruby):       49.0 i/s
+# Blank string (Crystal):       47.1 i/s - same-ish: difference falls within error
+
+# Warming up --------------------------------------
+#    abd vs abc (Ruby)     1.000  i/100ms
+# abd vs abc (Crystal)     1.000  i/100ms
+# Calculating -------------------------------------
+#    abd vs abc (Ruby)      1.367  (± 0.0%) i/s -      7.000  in   5.123125s
+# abd vs abc (Crystal)      2.750  (± 0.0%) i/s -     14.000  in   5.101675s
+
+# Comparison:
+# abd vs abc (Crystal):        2.7 i/s
+#    abd vs abc (Ruby):        1.4 i/s - 2.01x  (± 0.00) slower
+
+# Warming up --------------------------------------
+#  abc vs abcde (Ruby)     1.000  i/100ms
+# abc vs abcde (Crystal)
+#                          1.000  i/100ms
+# Calculating -------------------------------------
+#  abc vs abcde (Ruby)      0.901  (± 0.0%) i/s -      5.000  in   5.547298s
+# abc vs abcde (Crystal)
+#                           1.792  (± 0.0%) i/s -      9.000  in   5.023881s
+
+# Comparison:
+# abc vs abcde (Crystal):        1.8 i/s
+#  abc vs abcde (Ruby):        0.9 i/s - 1.99x  (± 0.00) slower
+
+# Warming up --------------------------------------
+# abcdefghi vs 0123456789 (Ruby)
+#                          1.000  i/100ms
+# abcdefghi vs 0123456789 (Crystal)
+#                          1.000  i/100ms
+# Calculating -------------------------------------
+# abcdefghi vs 0123456789 (Ruby)
+#                           0.196  (± 0.0%) i/s -      1.000  in   5.095124s
+# abcdefghi vs 0123456789 (Crystal)
+#                           0.441  (± 0.0%) i/s -      3.000  in   6.804299s
+
+# Comparison:
+# abcdefghi vs 0123456789 (Crystal):        0.4 i/s
+# abcdefghi vs 0123456789 (Ruby):        0.2 i/s - 2.25x  (± 0.00) slower
+
+# Warming up --------------------------------------
+# [whole alphabet] vs 012345 (Ruby)
+#                          1.000  i/100ms
+# [whole alphabet] vs 012345 (Crystal)
+#                          1.000  i/100ms
+# Calculating -------------------------------------
+# [whole alphabet] vs 012345 (Ruby)
+#                           0.123  (± 0.0%) i/s -      1.000  in   8.150069s
+# [whole alphabet] vs 012345 (Crystal)
+#                           0.291  (± 0.0%) i/s -      2.000  in   6.886041s
+
+# Comparison:
+# [whole alphabet] vs 012345 (Crystal):        0.3 i/s
+# [whole alphabet] vs 012345 (Ruby):        0.1 i/s - 2.37x  (± 0.00) slower

data/ext/levenshtein_str/Makefile

@@ -0,0 +1,15 @@
+CRYSTAL = crystal
+TARGET = ../../lib/levenshtein_str.bundle
+
+install: all
+
+all: shards $(TARGET)
+
+shards:
+	shards
+
+$(TARGET): ./src/levenshtein_str.cr
+	$(CRYSTAL) $< --link-flags "-dynamic -bundle -Wl,-undefined,dynamic_lookup" -o $(TARGET)
+
+clean:
+	rm -f ../../**/*.bundle*

data/ext/levenshtein_str/extconf.rb

@@ -0,0 +1,5 @@
+require "mkmf"
+find_executable("crystal") or abort <<~ERR
+  You need crystal installed to use this gem.
+  Please check out https://crystal-lang.org/ for information on how to install it.
+ERR

data/ext/levenshtein_str/shard.yml

@@ -0,0 +1,18 @@
+name: crysal_gem_template
+version: 0.1.0
+
+authors:
+  - johansenja
+
+targets:
+  levenshtein_str:
+    main: src/levenshtein_str.cr
+#
+# add dependencies and development_dependencies:
+# dependencies:
+#   pg:
+#     github: will/crystal-pg
+#     version: "~> 0.5"
+#
+# development_dependencies:
+#   ...

data/ext/levenshtein_str/src/levenshtein_str.cr

@@ -0,0 +1,39 @@
+require "./lib/cr_ruby"
+
+def levenshtein(first : String, second : String)
+  len1, len2 = first.size, second.size
+  return len1 if len2.zero?
+  return len2 if len1.zero?
+
+  matrix = [(0..len1).to_a]
+  (1..len2).each do |j|
+    matrix << [j] + [0] * len1
+  end
+
+  (1..len2).each do |i|
+    (1..len1).each do |j|
+      if first[j-1] == second[i-1]
+        matrix[i][j] = matrix[i-1][j-1]
+      else
+        matrix[i][j] = {
+          matrix[i-1][j],
+          matrix[i][j-1],
+          matrix[i-1][j-1]
+        }.min + 1
+      end
+    end
+  end
+  return matrix.last.last
+end
+
+def levenshtein(self : CrRuby::VALUE, other : CrRuby::VALUE)
+  levenshtein(String.from_ruby(self), String.from_ruby(other)).to_ruby
+end
+
+fun init = Init_levenshtein_str
+  GC.init
+  LibCrystalMain.__crystal_main(0, Pointer(Pointer(UInt8)).null)
+
+  string = CrRuby.rb_define_class("String", CrRuby.rb_cObject)
+  CrRuby.rb_define_method(string, "levenshtein", ->levenshtein(CrRuby::VALUE, CrRuby::VALUE), 1)
+end

data/ext/levenshtein_str/src/lib/cr_ruby.cr

@@ -0,0 +1,58 @@
+lib CrRuby
+  # every ruby Object is a VALUE
+  type VALUE = Void*
+  # ruby method typing
+  type METHOD_FUNC = VALUE, VALUE -> VALUE
+
+  # a ruby Class is a VALUE
+  $rb_cObject : VALUE
+
+  # a ruby Module is also a VALUE
+  $rb_mObject : VALUE
+
+  # convert Ruby str to C str
+  fun rb_str_new_cstr(str : UInt8*) : VALUE
+
+  # convert Ruby str to plain string
+  fun rb_str_to_str(value: VALUE) : VALUE
+
+  # convert plain string pointer to C string pointer
+  fun rb_string_value_cstr(value_ptr : VALUE*) : UInt8*
+
+  # ruby Integer to crystal int
+  fun rb_num2int(value : VALUE) : Int32
+
+  # crystal int to ruby Integer
+  fun rb_int2inum(value : Int32) : VALUE
+
+  # define ruby class in C
+  fun rb_define_class(name: UInt8*, super: VALUE) : VALUE
+
+  # define ruby module in C
+  fun rb_define_module(name: UInt8*) : VALUE
+
+  # define ruby method in C
+  fun rb_define_method(klass: VALUE, name: UInt8*, func: METHOD_FUNC, argc: Int32)
+end
+
+class String
+  def to_ruby
+    CrRuby.rb_str_new_cstr self
+  end
+
+  def self.from_ruby(ruby_str : CrRuby::VALUE)
+    plain_string = CrRuby.rb_str_to_str(ruby_str)
+    c_string = CrRuby.rb_string_value_cstr(pointerof(plain_string))
+    new(c_string)
+  end
+end
+
+struct Int32
+  def to_ruby
+    CrRuby.rb_int2inum(self)
+  end
+
+  def self.from_ruby(ruby_int : CrRuby::VALUE)
+    CrRuby.rb_num2int(ruby_int)
+  end
+end

data/lib/levenshtein_str/version.rb

@@ -0,0 +1,3 @@
+module LevenshteinStr
+  VERSION = "0.1.0"
+end

data/lib/levenshtein_str.rb

@@ -0,0 +1,2 @@
+require "levenshtein_str/version"
+require "levenshtein_str.bundle"

data/spec/spec_helper.rb

@@ -0,0 +1,15 @@
+load "Rakefile"
+
+RSpec.configure do |config|
+  config.before(:suite) do
+    Rake::Task["clean"].invoke
+    Rake::Task["compile"].invoke
+    require "levenshtein_str"
+  end
+
+  config.after(:suite) do
+    # for some reason, this task never runs if just `invoke`d
+    # Rake::Task["clean"].invoke
+    system "rake clean"
+  end
+end

data/spec/string_spec.rb

@@ -0,0 +1,41 @@
+require_relative "./spec_helper"
+
+RSpec.describe String do
+  describe "#levenshtein" do
+    context "with one empty string" do
+      it "returns the other string's length" do
+        expect("".levenshtein("abc")).to eq 3
+        expect("abc".levenshtein("")).to eq 3
+      end
+    end
+
+    context "with two equal strings" do
+      it "returns 0 if empty" do
+        expect("".levenshtein("")).to eq 0
+      end
+
+      it "returns 0 if not empty" do
+        expect("abc".levenshtein("abc")).to eq 0
+      end
+    end
+
+    context "with 2 same length strings" do
+      it "can accurately return the difference" do
+        # c => p
+        # d => w
+        # h => l
+        expect("abcdefgh".levenshtein("abpwefgl")).to eq 3
+      end
+    end
+
+    context "with 2 varying length strings" do
+      it "returns the length dfference if otherwise the same" do
+        expect("abcde".levenshtein("abc")).to eq 2
+      end
+
+      it "correctly evaluates complex strings" do
+        expect("abcdefghijklmnop".levenshtein("435hq09yrz")).to eq 15
+      end
+    end
+  end
+end

metadata

@@ -0,0 +1,75 @@
+--- !ruby/object:Gem::Specification
+name: levenshtein_str
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- johansenja
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2020-05-08 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: benchmark-ips
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.7'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.7'
+description: Use String#levenshtein(other_str) to get the levenshtein distance between
+  2 strings. Useful for measuring approximate string similarity, and fuzzy matching.
+email: 
+executables: []
+extensions:
+- ext/levenshtein_str/extconf.rb
+extra_rdoc_files: []
+files:
+- Gemfile
+- Gemfile.lock
+- README.md
+- Rakefile
+- benchmark
+- ext/levenshtein_str/Makefile
+- ext/levenshtein_str/extconf.rb
+- ext/levenshtein_str/shard.yml
+- ext/levenshtein_str/src/levenshtein_str.cr
+- ext/levenshtein_str/src/lib/cr_ruby.cr
+- lib/levenshtein_str.rb
+- lib/levenshtein_str/version.rb
+- spec/spec_helper.rb
+- spec/string_spec.rb
+homepage: https://github.com/johansenja/levenshtein_str
+licenses:
+- MIT
+metadata:
+  source_code_uri: https://github.com/johansenja/levenshtein_str
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.0.8
+signing_key: 
+specification_version: 4
+summary: Get the Levenshtein distance between two strings
+test_files:
+- spec/spec_helper.rb
+- spec/string_spec.rb