simhash 0.2.0 → 0.2.1

data/lib/simhash.rb

@@ -1,8 +1,12 @@
 $KCODE = 'u' 
+
+require 'active_support/core_ext/string/multibyte'
 require 'unicode'
+
 require 'string'
 require 'integer'
 require 'simhash/stopwords'
+
 begin
   require 'string_hashing'
 rescue LoadError
@@ -30,7 +34,7 @@ module Simhash
       # cutting stop-words
       token = token.split(" ").reject{ |w| Stopwords::ALL.index(" #{w} ") != nil }.join(" ") if options[:stop_words]
             
-      next if token.size.zero? || token.size < token_min_size
+      next if token.size.zero? || token.mb_chars.size < token_min_size
       hashed_token = token.send(hashing_method, hashbits).to_i
       hashbits.times do |i|
         v[i] += (hashed_token & masks[i]).zero? ? -1 : +1

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: simhash
 version: !ruby/object:Gem::Version 
-  hash: 23
+  hash: 21
   prerelease: false
   segments: 
   - 0
   - 2
-  - 0
-  version: 0.2.0
+  - 1
+  version: 0.2.1
 platform: ruby
 authors: 
 - Alex Gusev
@@ -15,7 +15,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-08-20 00:00:00 +04:00
+date: 2010-09-01 00:00:00 +04:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -34,6 +34,20 @@ dependencies:
         version: 0.3.1
   type: :runtime
   version_requirements: *id001
+- !ruby/object:Gem::Dependency 
+  name: activesupport
+  prerelease: false
+  requirement: &id002 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  type: :runtime
+  version_requirements: *id002
 description: Implementation of Charikar simhashes in Ruby
 email: alex.gusev@bookmate.ru
 executables: []