simhash 0.2.2 → 0.2.3

data/lib/simhash.rb

@@ -18,14 +18,29 @@ module Simhash
   
   def self.hash(tokens, options={})
     hashbits = options[:hashbits] || 64
-    token_min_size = options[:token_min_size].to_i
     hashing_method = options[:hashing_method] || DEFAULT_STRING_HASH_METHOD
-    stop_sentenses = options[:stop_sentenses]
-    
+        
     v = [0] * hashbits
     masks = v.dup
     masks.each_with_index {|e, i| masks[i] = (1 << i)}
     
+    self.each_filtered_token(tokens, options) do |token|
+      hashed_token = token.send(hashing_method, hashbits).to_i
+      hashbits.times do |i|
+        v[i] += (hashed_token & masks[i]).zero? ? -1 : +1
+      end
+    end
+   
+    fingerprint = 0
+
+    hashbits.times { |i| fingerprint += 1 << i if v[i] >= 0 }  
+      
+    fingerprint    
+  end
+  
+  def self.each_filtered_token(tokens, options={})
+    token_min_size = options[:token_min_size].to_i
+    stop_sentenses = options[:stop_sentenses]
     tokens.each do |token|
       # cutting punctuation (\302\240 is unbreakable space)
       token = token.gsub(/(\s|\d|\W|\302\240| *— *|[«»\…\-\–\—]| )+/u,' ') if !options[:preserve_punctuation]
@@ -39,17 +54,15 @@ module Simhash
       next if stop_sentenses && stop_sentenses.include?(" #{token} ")
             
       next if token.size.zero? || token.mb_chars.size < token_min_size
-      hashed_token = token.send(hashing_method, hashbits).to_i
-      hashbits.times do |i|
-        v[i] += (hashed_token & masks[i]).zero? ? -1 : +1
-      end
-    end
-   
-    fingerprint = 0
-
-    hashbits.times { |i| fingerprint += 1 << i if v[i] >= 0 }  
       
-    fingerprint    
+      yield token      
+    end
+  end
+  
+  def self.filtered_tokens(tokens, options={})
+    filtered_tokens = []
+    self.each_filtered_token(tokens, options) { |token| filtered_tokens << token }
+    filtered_tokens  
   end
   
   def self.hm

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: simhash
 version: !ruby/object:Gem::Version 
-  hash: 19
+  hash: 17
   prerelease: false
   segments: 
   - 0
   - 2
-  - 2
-  version: 0.2.2
+  - 3
+  version: 0.2.3
 platform: ruby
 authors: 
 - Alex Gusev
@@ -15,7 +15,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-09-07 00:00:00 +04:00
+date: 2011-01-20 00:00:00 +03:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency