simhash2 0.0.3 → 0.0.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 8e8190a872a7ec498e0a500482e8814758de6cb4
-  data.tar.gz: b0ea69a076ab1a99584010b3c85b6c3d17d37eea
+SHA256:
+  metadata.gz: ef3ff21591cb53fba63b03c7623c6cadb3874c6d2f67ac9e817c3533b9f9c3d8
+  data.tar.gz: aca99aa445defc207028a58948df20a293abdae66a2a3bf201a96bf9ccf19d49
 SHA512:
-  metadata.gz: 290b5f9daf27c8d4a138e26aabc062b4c8896fb6506bb41da13ea5c1e44095286e82b9b34780b189d22fdae1e4f4d34d7dc0ce6438b9c47539584b82fb23a541
-  data.tar.gz: 5b961fbf6271a50571d557f628ef111ca7f9683378869bc9443635cb678b0a06942424287e5e6103702074f604a8ac39a06d2a2120c15b3bbfc26ee77d1cc5ab
+  metadata.gz: f9d2b12eaad8fd994a3e2f68aef4f75aa0b34358f2ca685e33371374dadbc3041046acb4999073a90d1a7d8d4346dde3e9ed79d6f6448aead28420227b0057cc
+  data.tar.gz: 8f978c0061d29706363b8c8dc63efee5d7cc9403ca2482173b09ba11bed256e1c2e7255d5adbeb40407648ff5efd19f6095131582c6944d5a625ecbbeabf3afe

data/README.md

@@ -35,3 +35,32 @@ simhash2 = Simhash.generate(str2) # => 13921220612431195624
 Simhash.hamming_distance(simhash1, simhash2) # => 8
 ```
 
+## Performance
+
+Thanks to some performance optimizations by [JayTeeSF](https://github.com/JayTeeSF), this gem generally performs better than `bookmate/simhash`, especially when working with longer strings with lots of tokens.
+
+```ruby
+test_str = "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum."
+
+def test_simhash (x)
+  x.simhash # bookmate/simhash
+end
+
+def test_simhash2 (x)
+  Simhash.generate(x) # this gem
+end
+
+n = 5000
+Benchmark.bm do |x|
+  x.report("simhash") { for i in 1..n; test_simhash(test_str); end }
+  x.report("simhash2") { for i in 1..n; test_simhash2(test_str); end }
+end
+```
+
+Results:
+
+```
+       user     system      total        real
+simhash  5.109375   0.093750   5.203125 (  5.199069)
+simhash2  4.109375   0.000000   4.109375 (  4.108586)
+```

data/lib/simhash2.rb

@@ -1,4 +1,4 @@
-require 'simhash2/version'
+require_relative 'simhash2/version'
 
 module Simhash
   extend self
@@ -17,19 +17,20 @@ module Simhash
   end
 
   def generate(str, options = {})
+    # the split is how we get our tokens (or shingles)
+    # adjust that, if we want to use shingles
     generate_from_tokens(str.split(/\s+/), options)
   end
 
   def generate_from_tokens(tokens, options = {})
-    filter_tokens(tokens, OPTIONS.merge(options))
-
     v = [0] * HASHBITS
-
     masks = v.dup
     masks.each_with_index { |_e, i| masks[i] = (1 << i) }
 
-    hashes = tokens.map { |token| simple_string_hash(token, HASHBITS) }
-    hashes.each do |h|
+    filter_tokens(tokens, OPTIONS.merge(options)) do |token|
+      h = simple_string_hash(token, HASHBITS)
+      #warn "simple_string_hash (for: #{token.inspect}): #{h.inspect}"
+
       HASHBITS.times do |i|
         v[i] += (h & masks[i]).zero? ? -1 : +1
       end
@@ -65,12 +66,28 @@ module Simhash
     x.to_i
   end
 
-  def filter_tokens(tokens, options)
-    tokens.map! { |e| e.downcase.gsub(/\W+/, '') }
-    tokens.reject! { |e| e.nil? || e.length < options[:min_token_length] }
-    tokens.reject! { |e| options[:stop_words].include?(e) } unless options[:stop_words].nil? || options[:stop_words].empty?
-    tokens.map!(&:stem) if options[:stemming]
-    tokens.uniq! if options[:unique]
+  def filter_tokens(tokens, options, &block)
+    altered_tokens = []
+    tokens.each do |e|
+      new_e = e.downcase.gsub(/\W+/, '')
+      next if new_e.nil? || new_e.length < options[:min_token_length]
+      if options[:stop_words] && !options[:stop_words].empty?
+        next if options[:stop_words].include?(new_e)
+      end
+      if options[:stemming]
+        altered_tokens << new_e.stem
+      else
+        altered_tokens << new_e
+      end
+    end
+    altered_tokens.uniq! if options[:unique]
+
+    if block_given?
+      altered_tokens.each {|e| block[e] }
+    else
+      tokens.clear
+      altered_tokens.each {|e| tokens << e }
+      tokens
+    end
   end
-
 end

data/lib/simhash2/version.rb

@@ -1,3 +1,3 @@
 module Simhash
-  VERSION = '0.0.3'.freeze
+  VERSION = '0.0.4'.freeze
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: simhash2
 version: !ruby/object:Gem::Version
-  version: 0.0.3
+  version: 0.0.4
 platform: ruby
 authors:
 - Jonathan Wong
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-09-01 00:00:00.000000000 Z
+date: 2018-10-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rspec
@@ -79,7 +79,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.6.11
+rubygems_version: 2.7.6
 signing_key: 
 specification_version: 4
 summary: A rewrite of the 'simhash' gem, which is an implementation of Moses Charikar's