simhash 0.1.0 → 0.2.0

data/README.rdoc

@@ -0,0 +1,53 @@
+==Absctract
+
+This is implementation of {Moses Charikar's simhashes}[http://portal.acm.org/citation.cfm?id=509965] in Ruby.
+
+==Usage
+
+When you have a string and want to calculate it's simhash, you should
+
+	my_string.simhash
+  
+By default it will generate 64-bit integer - that is simhash for this string 
+
+It's always better to tokenize string before simhashing. It's as simple as
+
+	my_string.simhash(:split_by => / /)
+  
+This will generate 64-bit integer based, but will split string into words before. 
+It's handy when you need to calculate similarity of strings based on word usage.
+You can split string as you like: by letters/sentences/specific letter-combinations, etc.
+
+	my_string.simhash(:split_by => /./, :bitlength => 512)
+  
+Sometimes you might need longer simhash (finding similarity for very long strings is a good example). 
+You can set length of result hash by passing bitlength parameter. This example will return 512-bit simhash
+for your string splitted by sentences.
+
+==Advanced usage
+
+It's useful to clean your string before simhashing. But it's useful not to clean, too.
+
+Here are examples:
+
+	my_string.simhash(:stop_words => true) # here we clean
+
+This will find stop-words in your string and remove them before simhashing. Stop-words are "the", "not", "about", etc. 
+Currently we remove only Russian and English stop-words.
+
+	my_string.simhash(:preserve_punctuation => true) # here we not
+  
+This will not remove punctuation before simhashing. Yes, we remove all dots, commas, etc. after splitting string to words by default.
+Because different punctiation does not mean difference in general. If you not agree you can turn this default off.
+
+==Installation
+
+As usual:
+
+	gem install simhash
+  
+But if you have {GNU MP library}[http://gmplib.org/], simhash will work faster! To check out which version is used, type:
+
+	Simhash::DEFAULT_STRING_HASH_METHOD
+	
+It should return symbol. If symbol ends with "rb", your simhash is slow. If you want make it faster, install GNU MP.

data/Rakefile

@@ -1,6 +1,7 @@
 require 'rake'
 require 'rake/testtask'
 require 'rake/rdoctask'
+require 'rake/extensiontask'
 
 $LOAD_PATH << File.join(File.dirname(__FILE__), 'lib')
 require 'simhash'
@@ -8,6 +9,8 @@ require 'simhash'
 desc 'Default: run unit tests.'
 task :default => [:test]
 
+Rake::ExtensionTask.new('string_hashing')
+
 desc 'Test the simhash gem'
 Rake::TestTask.new(:test) do |t|
   t.libs << 'lib'
@@ -18,7 +21,7 @@ end
 desc 'Start an IRB session with all necessary files required.'
 task :shell do |t|
   chdir File.dirname(__FILE__)
-  exec 'irb -I lib/ -I lib/simhash -I lib/string -I lib/integer -r rubygems -r init'
+  exec 'irb -I lib/ -I lib/simhash -I lib/string -I lib/integer -r rubygems'
 end
 
 desc 'Build the gemspec.'

data/ext/string_hashing/extconf.rb

@@ -0,0 +1,48 @@
+require 'mkmf'
+
+extension_name = 'string_hashing'
+# should link against the libgmp library
+$LDFLAGS << ' -lgmp'
+
+# Sort out the universal vs. single-archicture build problems on MacOS X
+if RUBY_PLATFORM.include?( 'darwin' )
+  puts "MacOS X build: fixing architecture flags:"
+
+  commonflags = nil
+  if ENV['ARCHFLAGS']
+    puts "  using the value in ARCHFLAGS environment variable (%p)." % [ ENV['ARCHFLAGS'] ]
+    commonflags = ENV['ARCHFLAGS']
+  else
+    $stderr.puts %{
+    ===========   WARNING   ===========
+    
+    You are building this extension on OS X without setting the 
+    ARCHFLAGS environment variable. 
+    
+    If you are seeing this message, that means that the 
+    build will probably fail.
+
+    ===================================
+    }.gsub( /^\t+/, '  ' )
+  end
+
+  if commonflags
+    $CFLAGS.gsub!( /-arch\s+\S+ /, '' )
+    $LDFLAGS.gsub!( /-arch\s+\S+ /, '' )
+    CONFIG['LDSHARED'].gsub!( /-arch\s+\S+ /, '' )
+
+    $CFLAGS << ' ' << commonflags
+    $LDFLAGS << ' ' << commonflags
+    CONFIG['LDSHARED'] << ' ' << commonflags
+  end
+end
+
+if find_header("gmp.h")
+  $stderr.puts "Configuring extensions"
+  dir_config(extension_name)
+  create_makefile(extension_name)
+else
+  $stderr.puts "Skipping building of C extension"
+  # creating foo Makefile to avoid building stuff
+  File.open(File.join(File.dirname(__FILE__), "Makefile"), "w"){|f| f.write("all: \ninstall: \n")}
+end

data/ext/string_hashing/string_hashing.c

@@ -0,0 +1,63 @@
+#include "ruby.h"
+#include <gmp.h>
+#include <stdio.h>
+
+VALUE StringHashing = Qnil;
+
+void Init_string_hashing();
+
+VALUE method_hash_vl(VALUE self, VALUE bitlength);
+
+void Init_string_hashing() {
+	rb_define_method(rb_cString, "hash_vl", method_hash_vl, 1);
+}
+
+VALUE method_hash_vl(VALUE self, VALUE bitlength) {
+    int bl = NUM2INT(bitlength);
+    
+    // for hard typecasting
+    unsigned char one_char;
+    char* result;
+    result = malloc(bl*sizeof(char));
+	unsigned long long len = RSTRING(self)->len;
+    char *string = RSTRING(self)->ptr;
+
+	if(len == 0){ return 0; }
+
+	mpz_t x, mask, long_len;
+    mpz_init_set_ui (long_len, len);
+    one_char = RSTRING(self)->ptr[0];
+    mpz_init_set_ui (x, one_char << 7);
+    int m = 1000003; 
+    
+    // generating mask of length bitlength filled with 1
+    mpz_init (mask);
+    mpz_ui_pow_ui(mask, 2, bl);
+    mpz_sub_ui (mask, mask, 1);
+    
+    mpz_t computations, byte;
+    mpz_init(computations);
+    mpz_init2 (byte, 8);
+
+    int i = 0;
+    for(i; i < len; i++) {
+        one_char = string[i];
+        mpz_set_ui(byte, one_char);
+        mpz_mul_ui(computations, x, m);
+        mpz_xor(computations, computations, byte);
+        mpz_and (x, mask, computations);
+    }
+
+    mpz_xor(x, x, long_len);
+    mpz_get_str (result, 10, x);
+    VALUE res = rb_str_new2(result);
+
+    mpz_clear(x);
+    mpz_clear(byte);
+    mpz_clear(computations);
+    mpz_clear(mask);
+    mpz_clear(long_len);
+    free(result);
+    
+	return res;
+}

data/lib/simhash.rb

@@ -1,11 +1,21 @@
 $KCODE = 'u' 
-require 'active_support/core_ext/string/multibyte'
-require File.join(File.dirname(__FILE__), "simhash", "stopwords")
+require 'unicode'
+require 'string'
+require 'integer'
+require 'simhash/stopwords'
+begin
+  require 'string_hashing'
+rescue LoadError
+end
+
 
 module Simhash  
+  DEFAULT_STRING_HASH_METHOD = String.public_instance_methods.include?("hash_vl") ? :hash_vl : :hash_vl_rb
+  
   def self.hash(tokens, options={})
     hashbits = options[:hashbits] || 64
     token_min_size = options[:token_min_size].to_i
+    hashing_method = options[:hashing_method] || DEFAULT_STRING_HASH_METHOD
     
     v = [0] * hashbits
     masks = v.dup
@@ -15,15 +25,13 @@ module Simhash
       # cutting punctuation (\302\240 is unbreakable space)
       token = token.gsub(/(\s|\d|\W|\302\240| *— *|[«»\…\-\–\—]| )+/u,' ') if !options[:preserve_punctuation]
       
-      token = token.strip.mb_chars.downcase
+      token = Unicode::downcase(token.strip)
       
       # cutting stop-words
       token = token.split(" ").reject{ |w| Stopwords::ALL.index(" #{w} ") != nil }.join(" ") if options[:stop_words]
             
       next if token.size.zero? || token.size < token_min_size
-
-      hashed_token = token.hash_wl(hashbits)
-      bitmask = 0
+      hashed_token = token.send(hashing_method, hashbits).to_i
       hashbits.times do |i|
         v[i] += (hashed_token & masks[i]).zero? ? -1 : +1
       end
@@ -35,4 +43,8 @@ module Simhash
       
     fingerprint    
   end
+  
+  def self.hm
+    @@string_hash_method
+  end
 end

data/lib/string.rb

@@ -3,19 +3,18 @@ class String
     split_by = options.delete(:split_by) || " "
     Simhash.hash(self.split(split_by), options)
   end
-
   
-  # string hash of predefined length
-  def hash_wl(length)
+  def hash_vl_rb(length)
     return 0 if self == ""
 
     x = self[0] << 7
     m = 1000003
     mask = (1<<length) - 1
     self.each_byte{ |char| x = ((x * m) ^ char) & mask }
-        
+      
     x ^= self.size
     x = -2 if x == -1
     x
   end
+
 end

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: simhash
 version: !ruby/object:Gem::Version 
-  hash: 27
+  hash: 23
   prerelease: false
   segments: 
   - 0
-  - 1
+  - 2
   - 0
-  version: 0.1.0
+  version: 0.2.0
 platform: ruby
 authors: 
 - Alex Gusev
@@ -15,30 +15,45 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-08-17 00:00:00 +04:00
+date: 2010-08-20 00:00:00 +04:00
 default_executable: 
-dependencies: []
-
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: unicode
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 17
+        segments: 
+        - 0
+        - 3
+        - 1
+        version: 0.3.1
+  type: :runtime
+  version_requirements: *id001
 description: Implementation of Charikar simhashes in Ruby
 email: alex.gusev@bookmate.ru
 executables: []
 
-extensions: []
-
+extensions: 
+- ext/string_hashing/extconf.rb
 extra_rdoc_files: []
 
 files: 
-- README
+- README.rdoc
 - LICENSE
 - Rakefile
-- init.rb
 - lib/integer.rb
 - lib/simhash/stopwords/en.rb
 - lib/simhash/stopwords/ru.rb
 - lib/simhash/stopwords.rb
 - lib/simhash.rb
 - lib/string.rb
-- rails/init.rb
+- ext/string_hashing/extconf.rb
+- ext/string_hashing/string_hashing.c
 has_rdoc: true
 homepage: http://github.com/bookmate/simhash
 licenses: []

data/README

@@ -1 +0,0 @@
-Implementation of Charikar simhashes in Ruby

data/init.rb

@@ -1,3 +0,0 @@
-require File.join(File.dirname(__FILE__), "lib", "simhash")
-require File.join(File.dirname(__FILE__), "lib", "string")
-require File.join(File.dirname(__FILE__), "lib", "integer")

data/rails/init.rb

@@ -1,3 +0,0 @@
-require File.join(File.dirname(__FILE__), "..", "lib", "simhash")
-require File.join(File.dirname(__FILE__), "..", "lib", "string")
-require File.join(File.dirname(__FILE__), "..", "lib", "integer")