simhash 0.0.4 → 0.1.0

data/lib/simhash.rb

@@ -1,5 +1,3 @@
-#require 'rubygems'
-#require 'lingua/stemmer'
 $KCODE = 'u' 
 require 'active_support/core_ext/string/multibyte'
 require File.join(File.dirname(__FILE__), "simhash", "stopwords")
@@ -7,18 +5,23 @@ require File.join(File.dirname(__FILE__), "simhash", "stopwords")
 module Simhash  
   def self.hash(tokens, options={})
     hashbits = options[:hashbits] || 64
+    token_min_size = options[:token_min_size].to_i
     
     v = [0] * hashbits
     masks = v.dup
     masks.each_with_index {|e, i| masks[i] = (1 << i)}
     
     tokens.each do |token|
-      token = token.gsub(/(\s|\d|\W)+/u,' ').strip
+      # cutting punctuation (\302\240 is unbreakable space)
+      token = token.gsub(/(\s|\d|\W|\302\240| *— *|[«»\…\-\–\—]| )+/u,' ') if !options[:preserve_punctuation]
       
-      next if token.size < 2 
-      next if options[:stop_words] && Stopwords::ALL.index(" #{token.strip.mb_chars.downcase} ") != nil
+      token = token.strip.mb_chars.downcase
       
-      #token = Lingua.stemmer(token, :language => :ru)
+      # cutting stop-words
+      token = token.split(" ").reject{ |w| Stopwords::ALL.index(" #{w} ") != nil }.join(" ") if options[:stop_words]
+            
+      next if token.size.zero? || token.size < token_min_size
+
       hashed_token = token.hash_wl(hashbits)
       bitmask = 0
       hashbits.times do |i|

data/lib/string.rb

@@ -1,7 +1,9 @@
 class String
   def simhash(options={})
-    Simhash.hash(self.split, options)
+    split_by = options.delete(:split_by) || " "
+    Simhash.hash(self.split(split_by), options)
   end
+
   
   # string hash of predefined length
   def hash_wl(length)

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: simhash
 version: !ruby/object:Gem::Version 
-  hash: 23
+  hash: 27
   prerelease: false
   segments: 
   - 0
+  - 1
   - 0
-  - 4
-  version: 0.0.4
+  version: 0.1.0
 platform: ruby
 authors: 
 - Alex Gusev
@@ -15,7 +15,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-08-13 00:00:00 +04:00
+date: 2010-08-17 00:00:00 +04:00
 default_executable: 
 dependencies: []