string_metric 0.1.2 → 0.1.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 08ed461c4edc7dfd5e32fb34e93b13b4442c5167
-  data.tar.gz: 69d6cd7273238442f184d6a63fec949ffcc6f6db
+  metadata.gz: b2a2bb553d840612ee0f9bb1eb36f561e55e5d50
+  data.tar.gz: be1ba43ed05865948bf84e51b6eb9b85fabc170f
 SHA512:
-  metadata.gz: f5d7e8664514c5f31075755c3b09b2a7c4a0cd963601aefd26f1b928426dea2051a12b8abd039dddb0d8986fe25c6ce860d8c4a867757a4bf5b5a1883c6ace1e
-  data.tar.gz: d9b631aad62da262d7a9afec9c7c1295e8b238cbfe4ef432354e0a2c42d9bf3b777758cf00abe2b14b702069eeb335df505ba9db37faa165335d748f4d748f2e
+  metadata.gz: f449e8a3cd1303c9ee92bd71fe57bde264b03c201c85bfbe538353da26fbc3c91ad2d5394691535e6d2e59a025f391a5408b479182ba8fca2683aeada3aa2ff5
+  data.tar.gz: 52da6e7fd9076dd34c6b2b6325fdbea4e61a0dc37c38594ac76d3700adb61f10de960648a9b23dc8a9d730f5ea6bbcbab244a260371b3202be7fde2e680b4149

data/.gitignore

@@ -1,5 +1,7 @@
 *.gem
 *.rbc
+*.o
+*.so
 .bundle
 .config
 .ruby-version

data/.travis.yml

@@ -4,5 +4,3 @@ rvm:
   - "2.0.0"
   - "2.1.0"
   - "jruby-19mode"
-
-script: bundle exec rspec spec

data/Rakefile

@@ -1 +1,27 @@
 require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new(:spec)
+
+if RUBY_ENGINE == "ruby"
+  require "rake/extensiontask"
+
+  gem_name = "string_metric"
+  dir = "#{gem_name}/levenshtein"
+  spec = Gem::Specification.load("#{gem_name}.gemspec")
+
+  Rake::ExtensionTask.new do |ext|
+    ext.name = "trie_radix_tree_ext"
+    ext.ext_dir = "ext/#{dir}"
+    ext.lib_dir = "lib/#{dir}"
+    ext.gem_spec = spec
+  end
+
+  task :default do
+    Rake::Task["compile"].invoke
+    Rake::Task["spec"].invoke
+  end
+else
+  task default: :spec
+end
+

data/benchmarks/dictionary.rb

@@ -0,0 +1,44 @@
+require 'string_metric'
+require 'benchmark'
+require 'pp'
+
+Benchmark.bmbm(7) do |x|
+  options = {}
+  max_distance = 2
+
+  dict = []
+  trie = StringMetric::Levenshtein::TrieNode.new
+  File.open('/usr/share/dict/words', 'r').each_line do |line|
+    word = line.chomp
+    trie.insert(word)
+    dict << word
+  end
+
+  randomWords = []
+  File.open('spec/fixtures/dictionary_input.txt', 'r').each_line do |word|
+    randomWords << word.chomp
+  end
+
+  matrixResults = []
+  x.report("two_matrix_rows_v2 implementation") do
+    randomWords.each do |from|
+      dict.each do |to|
+        matrixResults << to if StringMetric::Levenshtein::IterativeWithTwoMatrixRowsOptimized.distance(from, to, options) <= max_distance
+      end
+    end
+  end
+
+  trieResults = []
+  x.report("trie_radix_tree implementation") do
+    randomWords.each do |from|
+      trieResults << StringMetric::Levenshtein::TrieRadixTree.distance(from, trie, max_distance: max_distance)
+    end
+  end
+
+  trieResultsExt = []
+  x.report("trie_radix_tree_ext implementation") do
+    randomWords.each do |from|
+      trieResultsExt << StringMetric::Levenshtein::TrieRadixTreeExt.distance(from, trie, max_distance: max_distance)
+    end
+  end
+end

data/ext/string_metric/levenshtein/extconf.rb

@@ -0,0 +1,3 @@
+require 'mkmf'
+
+create_makefile('string_metric/levenshtein/trie_radix_tree_ext')

data/ext/string_metric/levenshtein/trie_radix_tree_ext.c

@@ -0,0 +1,112 @@
+#include <ruby.h>
+
+void Init_trie_radix_tree_ext(void);
+void search_recursive(VALUE node, int letter, int *current_row, VALUE results);
+VALUE search_ext(VALUE self, VALUE _from, VALUE _from_len, VALUE trie_node,
+				VALUE _max_distance, VALUE _insertion_cost,
+				VALUE _deletion_cost, VALUE _substitution_cost);
+
+#define MIN2(a, b) (((a) < (b)) ? (a) : (b))
+#define MIN3(a, b, c) (MIN2(MIN2((a), (b)), (c)))
+
+#define MALLOC_W(ptr, size) do { \
+	ptr = malloc(size);          \
+	if (!ptr)                    \
+		rb_memerror();           \
+} while (0)
+
+// Declare the variables that don't change as global, so that we don't have to pass them around
+// in our recursive function and increase the stack frame unnecessarily
+int *from, from_len;
+int max_distance, insertion_cost, deletion_cost, substitution_cost;
+
+void Init_trie_radix_tree_ext(void) {
+
+	VALUE StringMetric     = rb_define_module("StringMetric");
+	VALUE Levenshtein      = rb_define_module_under(StringMetric, "Levenshtein");
+	VALUE TrieRadixTreeExt = rb_define_class_under(Levenshtein, "TrieRadixTreeExt", rb_cObject);
+
+	rb_define_singleton_method(TrieRadixTreeExt, "trie_ext", search_ext, 7);
+}
+
+VALUE search_ext(VALUE self, VALUE _from, VALUE _from_len, VALUE trie_node,
+				VALUE _max_distance,  VALUE _insertion_cost,
+				VALUE _deletion_cost, VALUE _substitution_cost)
+{
+	int i, *current_row;
+	VALUE results, letter, node, children, children_keys;
+
+	// Convert from ruby types
+	max_distance      = FIX2INT(_max_distance);
+	insertion_cost    = FIX2INT(_insertion_cost);
+	deletion_cost     = FIX2INT(_deletion_cost);
+	substitution_cost = FIX2INT(_substitution_cost);
+	from_len          = FIX2INT(_from_len);
+
+	// The '_from' word is passed as an array of codepoints. Allocate memory and populate the C array
+	MALLOC_W(from, from_len * sizeof(int));
+	for (i = 0; i < from_len; i++)
+		from[i] = FIX2INT(rb_ary_entry(_from, i));
+
+	// Create a hash to store the results and return it to ruby when we are done
+	results = rb_hash_new();
+
+	MALLOC_W(current_row, (from_len + 1) * sizeof(int));
+	for (i = 0; i <= from_len; i++)
+		current_row[i] = i;
+
+	// Extract the hash from trie_node object and get an array of keys
+	children = rb_funcall(trie_node, rb_intern("children"), 0);
+	children_keys = rb_funcall(children, rb_intern("keys"), 0);
+
+	for (i = 0; i < RARRAY_LEN(children_keys); i++) {
+		letter = rb_ary_entry(children_keys, i);
+		node = rb_hash_aref(children, letter);
+		search_recursive(node, FIX2INT(letter), current_row, results);
+	}
+	free(from);
+	free(current_row);
+	return results;
+}
+
+void search_recursive(VALUE node, int letter, int *previous_row, VALUE results) {
+
+	int i, min, columns, distance, *current_row;
+	int cost, insert_cost, delete_cost, replace_cost;
+	VALUE word, codepoint, children, children_keys;
+
+	columns = from_len + 1;
+	MALLOC_W(current_row, columns * sizeof(int));
+	current_row[0] = previous_row[0] + 1;
+
+	for (i = 1; i < columns; i++) {
+		cost = (from[i - 1] == letter) ? 0 : substitution_cost;
+		insert_cost  = current_row[i - 1]  + insertion_cost;
+		delete_cost  = previous_row[i]     + deletion_cost;
+		replace_cost = previous_row[i - 1] + cost;
+
+		current_row[i] = MIN3(insert_cost, delete_cost, replace_cost);
+	}
+	distance = current_row[columns - 1];
+	word = rb_funcall(node, rb_intern("word"), 0);
+
+	if (distance <= max_distance && word != Qnil)
+		rb_hash_aset(results, word, INT2FIX(distance));
+
+	min = current_row[0];
+	for (i = 1; i < columns; i++)
+		if (current_row[i] < min)
+			min = current_row[i];
+
+	if (min <= max_distance) {
+		children = rb_funcall(node, rb_intern("children"), 0);
+		children_keys = rb_funcall(children, rb_intern("keys"), 0);
+
+		for (i = 0; i < RARRAY_LEN(children_keys); i++) {
+			codepoint = rb_ary_entry(children_keys, i);
+			node = rb_hash_aref(children, codepoint);
+			search_recursive(node, FIX2INT(codepoint), current_row, results);
+		}
+	}
+	free(current_row);
+}

data/lib/string_metric/levenshtein/trie_node.rb

@@ -0,0 +1,23 @@
+# coding: utf-8
+
+module StringMetric
+  module Levenshtein
+    class TrieNode
+      attr_accessor :word, :children
+
+      def initialize
+        @word = nil
+        @children = {}
+      end
+
+      def insert(word)
+        node = self
+        word.codepoints.each do |char|
+          node.children[char] = TrieNode.new unless node.children.key?(char)
+          node = node.children[char]
+        end
+        node.word = word
+      end
+    end
+  end
+end

data/lib/string_metric/levenshtein/trie_radix_tree.rb

@@ -0,0 +1,49 @@
+# coding: utf-8
+
+module StringMetric
+  module Levenshtein
+    class TrieRadixTree
+      def self.distance(from, node, options = {})
+
+        @max_distance      = options[:max_distance]      || 0
+        @insertion_cost    = options[:insertion_cost]    || 1
+        @deletion_cost     = options[:deletion_cost]     || 1
+        @substitution_cost = options[:substitution_cost] || 1
+
+        results = []
+        word = from.codepoints
+        currentRow = (0..word.length).to_a
+
+        node.children.keys.each do |letter|
+          searchRecursive(node.children[letter], letter, word, currentRow, results)
+        end
+
+        results
+      end
+
+      def self.searchRecursive(node, letter, word, previousRow, results)
+        columns = word.length + 1
+        currentRow = [previousRow[0] + 1]
+
+        (1...columns).each do |column|
+          insertCost = currentRow[column - 1] + @insertion_cost
+          deleteCost = previousRow[column] + @deletion_cost
+          cost = (word[column - 1] == letter) ? 0 : @substitution_cost
+          replaceCost = previousRow[column - 1] + cost
+
+          currentRow << [insertCost, deleteCost, replaceCost].min
+        end
+
+        if currentRow.last <= @max_distance && !node.word.nil?
+          results << [node.word, currentRow.last]
+        end
+
+        if currentRow.min <= @max_distance
+          node.children.keys.each do |letter|
+            searchRecursive(node.children[letter], letter, word, currentRow, results)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/string_metric/levenshtein/trie_radix_tree_ext.rb

@@ -0,0 +1,20 @@
+# coding: utf-8
+
+require_relative 'trie_radix_tree_ext.so'
+
+module StringMetric
+  module Levenshtein
+    class TrieRadixTreeExt
+      def self.distance(from, trieNode, options = {})
+
+        max_distance      = options[:max_distance]      || 0
+        insertion_cost    = options[:insertion_cost]    || 1
+        deletion_cost     = options[:deletion_cost]     || 1
+        substitution_cost = options[:substitution_cost] || 1
+
+        trie_ext(from.codepoints, from.length, trieNode, max_distance,
+                 insertion_cost, deletion_cost, substitution_cost)
+      end
+    end
+  end
+end

data/lib/string_metric/levenshtein.rb

@@ -5,6 +5,9 @@ require_relative "levenshtein/iterative_with_two_matrix_rows"
 require_relative "levenshtein/iterative_with_two_matrix_rows_optimized"
 require_relative "levenshtein/iterative_with_full_matrix"
 require_relative "levenshtein/recursive"
+require_relative "levenshtein/trie_node"
+require_relative "levenshtein/trie_radix_tree"
+require_relative "levenshtein/trie_radix_tree_ext"
 
 module StringMetric
   # Levenshtein Distance implementation

data/lib/string_metric/version.rb

@@ -1,3 +1,3 @@
 module StringMetric
-  VERSION = "0.1.2"
+  VERSION = "0.1.3"
 end

data/spec/fixtures/dictionary_input.txt

@@ -0,0 +1,10 @@
+aliency
+prostatauxe
+Herbartian
+womanize
+unviolent
+disguised
+preanimism
+birdling
+geognosy
+daut

data/string_metric.gemspec

@@ -24,6 +24,9 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "text", "~> 1.2.3"
 
   if RUBY_ENGINE == "ruby"
+    spec.add_development_dependency "rake-compiler", "~> 0.9.2"
+    spec.extensions << "ext/#{spec.name}/levenshtein/extconf.rb"
+
     if RUBY_VERSION > "1.9.3"
       spec.add_development_dependency "pry-byebug", "~> 1.2.1"
     else

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: string_metric
 version: !ruby/object:Gem::Version
-  version: 0.1.2
+  version: 0.1.3
 platform: ruby
 authors:
 - Giorgos Tsiftsis
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-04-13 00:00:00.000000000 Z
+date: 2015-05-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -66,6 +66,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 1.2.3
+- !ruby/object:Gem::Dependency
+  name: rake-compiler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.9.2
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.9.2
 - !ruby/object:Gem::Dependency
   name: pry-byebug
   requirement: !ruby/object:Gem::Requirement
@@ -84,7 +98,8 @@ description: A simple library with String Metric algorithms
 email:
 - giorgos.tsiftsis@skroutz.gr
 executables: []
-extensions: []
+extensions:
+- ext/string_metric/levenshtein/extconf.rb
 extra_rdoc_files: []
 files:
 - ".gitignore"
@@ -94,7 +109,10 @@ files:
 - LICENSE.txt
 - README.md
 - Rakefile
+- benchmarks/dictionary.rb
 - benchmarks/levenshtein.rb
+- ext/string_metric/levenshtein/extconf.rb
+- ext/string_metric/levenshtein/trie_radix_tree_ext.c
 - lib/string_metric.rb
 - lib/string_metric/levenshtein.rb
 - lib/string_metric/levenshtein/experiment.rb
@@ -102,7 +120,11 @@ files:
 - lib/string_metric/levenshtein/iterative_with_two_matrix_rows.rb
 - lib/string_metric/levenshtein/iterative_with_two_matrix_rows_optimized.rb
 - lib/string_metric/levenshtein/recursive.rb
+- lib/string_metric/levenshtein/trie_node.rb
+- lib/string_metric/levenshtein/trie_radix_tree.rb
+- lib/string_metric/levenshtein/trie_radix_tree_ext.rb
 - lib/string_metric/version.rb
+- spec/fixtures/dictionary_input.txt
 - spec/fixtures/levenshtein.csv
 - spec/lib/levenshtein/experiment_spec.rb
 - spec/lib/levenshtein/iterative_with_full_matric_spec.rb
@@ -138,6 +160,7 @@ signing_key:
 specification_version: 4
 summary: A simple library with String Metric algorithms
 test_files:
+- spec/fixtures/dictionary_input.txt
 - spec/fixtures/levenshtein.csv
 - spec/lib/levenshtein/experiment_spec.rb
 - spec/lib/levenshtein/iterative_with_full_matric_spec.rb