mort666_simhash 0.2.7

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 3b3b62e04b6e50547de7e7edf3441da79ce41869
+  data.tar.gz: dde5d235ee1d2bbeaa9304fe235768883c6525cf
+SHA512:
+  metadata.gz: c333facb22f608bf1caec2de3023f30ccc552f116a323063819dfcfe2bc49d1bc397a3f930d3ac8caa4e19d1f6308542096b4732e5ae308a9c1dc241083674ee
+  data.tar.gz: 5c1ef0e033e82009858a7136e23c01936ce7083c551783c8987f36a31a294e0a1babdc82e2bd3a943c60a444bb724fe09affc2b1998495b2172dd8d58b1e06cf

data/LICENSE

@@ -0,0 +1,28 @@
+
+LICENSE
+
+The MIT License
+
+Copyright (c) 2002 Charikar, Simhash algorythm
+Copyright (c) 2009 Andre Hagenbruch, Python implementation
+Copyright (c) 2010 Bookmate.ru
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+
+

data/README.rdoc

@@ -0,0 +1,53 @@
+==Absctract
+
+This is implementation of {Moses Charikar's simhashes}[http://portal.acm.org/citation.cfm?id=509965] in Ruby.
+
+==Usage
+
+When you have a string and want to calculate it's simhash, you should
+
+	my_string.simhash
+  
+By default it will generate 64-bit integer - that is simhash for this string 
+
+It's always better to tokenize string before simhashing. It's as simple as
+
+	my_string.simhash(:split_by => / /)
+  
+This will generate 64-bit integer based, but will split string into words before. 
+It's handy when you need to calculate similarity of strings based on word usage.
+You can split string as you like: by letters/sentences/specific letter-combinations, etc.
+
+	my_string.simhash(:split_by => /./, :hashbits => 512)
+  
+Sometimes you might need longer simhash (finding similarity for very long strings is a good example). 
+You can set length of result hash by passing hashbits parameter. This example will return 512-bit simhash
+for your string splitted by sentences.
+
+==Advanced usage
+
+It's useful to clean your string before simhashing. But it's useful not to clean, too.
+
+Here are examples:
+
+	my_string.simhash(:stop_words => true) # here we clean
+
+This will find stop-words in your string and remove them before simhashing. Stop-words are "the", "not", "about", etc. 
+Currently we remove only Russian and English stop-words.
+
+	my_string.simhash(:preserve_punctuation => true) # here we not
+  
+This will not remove punctuation before simhashing. Yes, we remove all dots, commas, etc. after splitting string to words by default.
+Because different punctiation does not mean difference in general. If you not agree you can turn this default off.
+
+==Installation
+
+As usual:
+
+	gem install simhash
+  
+But if you have {GNU MP library}[http://gmplib.org/], simhash will work faster! To check out which version is used, type:
+
+	Simhash::DEFAULT_STRING_HASH_METHOD
+	
+It should return symbol. If symbol ends with "rb", your simhash is slow. If you want make it faster, install GNU MP.

data/Rakefile

@@ -0,0 +1,51 @@
+require 'rake'
+require 'rake/testtask'
+require 'rdoc/task'
+require "rake/extensiontask"
+
+$LOAD_PATH << File.join(File.dirname(__FILE__), 'lib')
+require 'simhash'
+
+desc 'Default: run unit tests.'
+task :default => [:compile, :test]
+
+Rake::ExtensionTask.new('string_hashing')
+
+Rake::Task[:test].prerequisites << :compile
+
+desc 'Test the simhash gem'
+Rake::TestTask.new(:test) do |t|
+  t.libs << '.'
+  t.pattern = 'test/**/*_test.rb'
+  t.verbose = true
+end
+
+desc 'Start an IRB session with all necessary files required.'
+task :shell do |t|
+  chdir File.dirname(__FILE__)
+  exec 'irb -I lib/ -I lib/simhash -I lib/string -I lib/integer -r rubygems'
+end
+
+desc 'Build the gemspec.'
+task :gemspec do |t|
+  exec 'gem build mort666-simhash.gemspec'
+end
+
+desc "Print a list of the files to be put into the gem"
+task :manifest do
+  spec.files.each do |file|
+    puts file
+  end
+end
+
+desc "Generate a gemspec file for GitHub"
+task :gemspec do
+  File.open("#{spec.name}.gemspec", 'w') do |f|
+    f.write spec.to_ruby
+  end
+end
+
+desc "Build the gem into the current directory"
+task :gem => :gemspec do
+  `gem build #{spec.name}.gemspec`
+end

data/ext/string_hashing/extconf.rb

@@ -0,0 +1,48 @@
+require 'mkmf'
+
+extension_name = 'string_hashing'
+# should link against the libgmp library
+$LDFLAGS << ' -lgmp'
+
+# Sort out the universal vs. single-archicture build problems on MacOS X
+if RUBY_PLATFORM.include?( 'darwin' )
+  puts "MacOS X build: fixing architecture flags:"
+
+  commonflags = nil
+  if ENV['ARCHFLAGS']
+    puts "  using the value in ARCHFLAGS environment variable (%p)." % [ ENV['ARCHFLAGS'] ]
+    commonflags = ENV['ARCHFLAGS']
+  else
+    $stderr.puts %{
+    ===========   WARNING   ===========
+    
+    You are building this extension on OS X without setting the 
+    ARCHFLAGS environment variable. 
+    
+    If you are seeing this message, that means that the 
+    build will probably fail.
+
+    ===================================
+    }.gsub( /^\t+/, '  ' )
+  end
+
+  if commonflags
+    $CFLAGS.gsub!( /-arch\s+\S+ /, '' )
+    $LDFLAGS.gsub!( /-arch\s+\S+ /, '' )
+    CONFIG['LDSHARED'].gsub!( /-arch\s+\S+ /, '' )
+
+    $CFLAGS << ' ' << commonflags
+    $LDFLAGS << ' ' << commonflags
+    CONFIG['LDSHARED'] << ' ' << commonflags
+  end
+end
+
+if (find_header("gmp.h") rescue false)
+  $stderr.puts "Configuring extensions"
+  dir_config(extension_name)
+  create_makefile(extension_name)
+else
+  $stderr.puts "Skipping building of C extension"
+  # creating foo Makefile to avoid building stuff
+  File.open(File.join(File.dirname(__FILE__), "Makefile"), "w"){|f| f.write("all: \ninstall: \n")}
+end

data/ext/string_hashing/string_hashing.c

@@ -0,0 +1,64 @@
+#include "ruby.h"
+#include <gmp.h>
+#include <stdio.h>
+
+VALUE StringHashing = Qnil;
+
+void Init_string_hashing();
+
+VALUE method_hash_vl(VALUE self, VALUE bitlength);
+
+void Init_string_hashing() {
+	rb_define_method(rb_cString, "hash_vl", method_hash_vl, 1);
+}
+
+VALUE method_hash_vl(VALUE self, VALUE bitlength) {
+    int bl = NUM2INT(bitlength);
+    
+    // for hard typecasting
+    unsigned char one_char;
+    char* result;
+    result = malloc(bl*sizeof(char));
+    unsigned long long len = RSTRING_LEN(self);
+    char *string = RSTRING_PTR(self);
+
+    if(len == 0){ return 0; }
+
+    mpz_t x, mask, long_len;
+    mpz_init_set_ui (long_len, len);
+    one_char = RSTRING_PTR(self)[0];
+    mpz_init_set_ui (x, one_char << 7);
+    int m = 1000003; 
+    
+    // generating mask of length bitlength filled with 1
+    mpz_init (mask);
+    mpz_ui_pow_ui(mask, 2, bl);
+    mpz_sub_ui (mask, mask, 1);
+    
+    mpz_t computations, byte;
+    mpz_init(computations);
+    mpz_init2 (byte, 8);
+
+    int i = 0;
+    for(i; i < len; i++) {
+        one_char = string[i];
+        mpz_set_ui(byte, one_char);
+        mpz_mul_ui(computations, x, m);
+        mpz_xor(computations, computations, byte);
+        mpz_and (x, mask, computations);
+    }
+    
+    mpz_xor(x, x, long_len);
+    //gmp_printf ("C xored x is %Zd\n", x);
+    mpz_get_str (result, 10, x);
+    VALUE res = rb_str_new2(result);
+
+    mpz_clear(x);
+    mpz_clear(byte);
+    mpz_clear(computations);
+    mpz_clear(mask);
+    mpz_clear(long_len);
+    free(result);
+    
+    return res;
+}

data/lib/integer.rb

@@ -0,0 +1,16 @@
+class Integer
+  # Hamming distance – number of different bits in same positions
+  # H(1001, 1110) = 3 
+  # H(1001, 1000) = 1
+  def hamming_distance_to(integer)
+    total = 0
+    difference = self ^ integer
+
+    while difference > 0 do
+      total += 1 if (difference & 1).nonzero?
+      difference >>= 1
+    end
+    
+    total
+  end
+end

data/lib/simhash/stopwords/en.rb

@@ -0,0 +1,5 @@
+module Simhash
+  module Stopwords
+    EN = " a able about above abst accordance according accordingly across act actually added adj adopted affected affecting affects after afterwards again against ah all almost alone along already also although always am among amongst an and announce another any anybody anyhow anymore anyone anything anyway anyways anywhere apparently approximately are aren arent arise around as aside ask asking at auth available away awfully b back be became because become becomes becoming been before beforehand begin beginning beginnings begins behind being believe below beside besides between beyond biol both brief briefly but by c ca came can cannot can't cause causes certain certainly co com come comes contain containing contains could couldnt d date did didn't different do does doesn't doing done don't down downwards due during e each ed edu effect eg eight eighty either else elsewhere end ending enough especially et et-al etc even ever every everybody everyone everything everywhere ex except f far few ff fifth first five fix followed following follows for former formerly forth found four from further furthermore g gave get gets getting give given gives giving go goes gone got gotten h had happens hardly has hasn't have haven't having he hed hence her here hereafter hereby herein heres hereupon hers herself hes hi hid him himself his hither home how howbeit however hundred i id ie if i'll im immediate immediately importance important in inc indeed index information instead into invention inward is isn't it itd it'll its itself i've j just k keep keeps kept keys kg km know known knows l largely last lately later latter latterly least less lest let lets like liked likely line little 'll look looking looks ltd m made mainly make makes many may maybe me mean means meantime meanwhile merely mg might million miss ml more moreover most mostly mr mrs much mug must my myself n na name namely nay nd near nearly necessarily necessary need needs neither never nevertheless new next nine ninety no nobody non none nonetheless noone nor normally nos not noted nothing now nowhere o obtain obtained obviously of off often oh ok okay old omitted on once one ones only onto or ord other others otherwise ought our ours ourselves out outside over overall owing own p page pages part particular particularly past per perhaps placed please plus poorly possible possibly potentially pp predominantly present previously primarily probably promptly proud provides put q que quickly quite qv r ran rather rd re readily really recent recently ref refs regarding regardless regards related relatively research respectively resulted resulting results right run s said same saw say saying says sec section see seeing seem seemed seeming seems seen self selves sent seven several shall she shed she'll shes should shouldn't show showed shown showns shows significant significantly similar similarly since six slightly so some somebody somehow someone somethan something sometime sometimes somewhat somewhere soon sorry specifically specified specify specifying state states still stop strongly sub substantially successfully such sufficiently suggest sup sure t take taken taking tell tends th than thank thanks thanx that that'll thats that've the their theirs them themselves then thence there thereafter thereby thered therefore therein there'll thereof therere theres thereto thereupon there've these they theyd they'll theyre they've think this those thou though thoughh thousand throug through throughout thru thus til tip to together too took toward towards tried tries truly try trying ts twice two u un under unfortunately unless unlike unlikely until unto up upon ups us use used useful usefully usefulness uses using usually v value various 've very via viz vol vols vs w want wants was wasn't way we wed welcome we'll went were weren't we've what whatever what'll whats when whence whenever where whereafter whereas whereby wherein wheres whereupon wherever whether which while whim whither who whod whoever whole who'll whom whomever whos whose why widely willing wish with within without won't words world would wouldn't www x y yes yet you youd you'll your youre yours yourself yourselves you've z zero "
+  end
+end

data/lib/simhash/stopwords/ru.rb

@@ -0,0 +1,6 @@
+# encoding: utf-8
+module Simhash
+  module Stopwords
+    RU = " а е и ж м о на не ни об но он мне мои мож она они оно мной много многочисленное многочисленная многочисленные многочисленный мною мой мог могут можно может можхо мор моя моё мочь над нее оба нам нем нами ними мимо немного одной одного менее однажды однако меня нему меньше ней наверху него ниже мало надо один одиннадцать одиннадцатый назад наиболее недавно миллионов недалеко между низко меля нельзя нибудь непрерывно наконец никогда никуда нас наш нет нею неё них мира наша наше наши ничего начала нередко несколько обычно опять около мы ну нх от отовсюду особенно нужно очень отсюда в во вон вниз внизу вокруг вот восемнадцать восемнадцатый восемь восьмой вверх вам вами важное важная важные важный вдали везде ведь вас ваш ваша ваше ваши впрочем весь вдруг вы все второй всем всеми времени время всему всего всегда всех всею всю вся всё всюду г год говорил говорит года году где да ее за из ли же им до по ими под иногда довольно именно долго позже более должно пожалуйста значит иметь больше пока ему имя пор пора потом потому после почему почти посреди ей два две двенадцать двенадцатый двадцать двадцатый двух его дел или без день занят занята занято заняты действительно давно девятнадцать девятнадцатый девять девятый даже алло жизнь далеко близко здесь дальше для лет зато даром первый перед затем зачем лишь десять десятый ею её их бы еще при был про процентов против просто бывает бывь если люди была были было будем будет будете будешь прекрасно буду будь будто будут ещё пятнадцать пятнадцатый друго другое другой другие другая других есть пять быть лучше пятый к ком конечно кому кого когда которой которого которая которые который которых кем каждое каждая каждые каждый кажется как какой какая кто кроме куда кругом с т у я та те уж со то том снова тому совсем того тогда тоже собой тобой собою тобою сначала только уметь тот тою хорошо хотеть хочешь хоть хотя свое свои твой своей своего своих свою твоя твоё раз уже сам там тем чем сама сами теми само рано самом самому самой самого семнадцать семнадцатый самим самими самих саму семь чему раньше сейчас чего сегодня себе тебе сеаой человек разве теперь себя тебя седьмой спасибо слишком так такое такой такие также такая сих тех чаще четвертый через часто шестой шестнадцать шестнадцатый шесть четыре четырнадцать четырнадцатый сколько сказал сказала сказать ту ты три эта эти что это чтоб этом этому этой этого чтобы этот стал туда этим этими рядом тринадцать тринадцатый этих третий тут эту суть чуть тысяч "
+  end
+end

data/lib/simhash/stopwords.rb

@@ -0,0 +1,8 @@
+require File.join(File.dirname(__FILE__), "stopwords", "en")
+require File.join(File.dirname(__FILE__), "stopwords", "ru")
+
+module Simhash
+  module Stopwords
+    ALL = RU + EN
+  end
+end

data/lib/simhash.rb

@@ -0,0 +1,93 @@
+# encoding: utf-8
+
+require 'active_support/core_ext/string/multibyte'
+require 'unicode'
+
+require 'string'
+require 'integer'
+require 'simhash/stopwords'
+
+begin
+  require 'string_hashing'
+rescue LoadError
+end
+
+module Simhash  
+  DEFAULT_STRING_HASH_METHOD = String.public_instance_methods.include?("hash_vl") ? :hash_vl : :hash_vl_rb
+  PUNCTUATION_REGEXP = if RUBY_VERSION >= "1.9"
+    /(\s|\d|[^\p{L}]|\302\240| *— *|[«»…\-–—]| )+/u
+  else
+    /(\s|\d|\W|\302\240| *— *|[«»…\-–—]| )+/u
+  end
+  
+  # Compare calculates the Hamming distance between two 64-bit integers
+  #
+  # Currently, this is calculated using the Kernighan method [1]. Other methods
+  # exist which may be more efficient and are worth exploring at some point
+  #
+  # [1] http://graphics.stanford.edu/~seander/bithacks.html#CountBitsSetKernighan
+  def self.compare(a, b)
+    v = a ^ b
+    c = 0
+    while v != 0 do
+      v &= v - 1
+      c += 1
+    end
+
+    return c
+  end
+
+  def self.hash(tokens, options={})
+    hashbits = options[:hashbits] || 64
+    hashing_method = options[:hashing_method] || DEFAULT_STRING_HASH_METHOD
+        
+    v = [0] * hashbits
+    masks = v.dup
+    masks.each_with_index {|e, i| masks[i] = (1 << i)}
+    
+    self.each_filtered_token(tokens, options) do |token|
+      hashed_token = token.send(hashing_method, hashbits).to_i
+      hashbits.times do |i|
+        v[i] += (hashed_token & masks[i]).zero? ? -1 : +1
+      end
+    end
+   
+    fingerprint = 0
+
+    hashbits.times { |i| fingerprint += 1 << i if v[i] >= 0 }  
+      
+    fingerprint    
+  end
+  
+  def self.each_filtered_token(tokens, options={})
+    token_min_size = options[:token_min_size].to_i
+    stop_sentenses = options[:stop_sentenses]
+    tokens.each do |token|
+      # cutting punctuation (\302\240 is unbreakable space)
+      # Moved up to string class
+      # token = token.gsub(PUNCTUATION_REGEXP, ' ') if !options[:preserve_punctuation]
+      
+      token = Unicode::downcase(token.strip)
+      
+      # cutting stop-words
+      token = token.split(" ").reject{ |w| Stopwords::ALL.index(" #{w} ") != nil }.join(" ") if options[:stop_words]
+    
+      # cutting stop-sentenses
+      next if stop_sentenses && stop_sentenses.include?(" #{token} ")
+            
+      next if token.size.zero? || token.mb_chars.size < token_min_size
+      
+      yield token      
+    end
+  end
+  
+  def self.filtered_tokens(tokens, options={})
+    filtered_tokens = []
+    self.each_filtered_token(tokens, options) { |token| filtered_tokens << token }
+    filtered_tokens  
+  end
+  
+  def self.hm
+    @@string_hash_method
+  end
+end

data/lib/string.rb

@@ -0,0 +1,28 @@
+class String
+  def simhash(options={})
+    split_by = options.delete(:split_by) || " "
+
+    # Do the punctuation clean before the split.. 
+    # You could argue this is not preserving the meaning doing so actually preserves the edge case where a hyphen is removed
+    # resulting hash does not match the same string with a space in there before the split
+    if options[:preserve_punctuation]
+      Simhash.hash(self.split(split_by), options)
+    else
+      Simhash.hash(self.gsub(Simhash::PUNCTUATION_REGEXP, ' ') .split(split_by), options)
+    end
+  end
+  
+  def hash_vl_rb(length)
+    return 0 if self == ""
+
+    x = self.bytes.first << 7
+    m = 1000003
+    mask = (1<<length) - 1
+    self.each_byte{ |char| x = ((x * m) ^ char.to_i) & mask }
+
+    x ^= self.bytes.count
+    x = -2 if x == -1
+    x
+  end
+
+end

metadata

@@ -0,0 +1,142 @@
+--- !ruby/object:Gem::Specification
+name: mort666_simhash
+version: !ruby/object:Gem::Version
+  version: 0.2.7
+platform: ruby
+authors:
+- Alex Gusev
+- Stephen Kapp
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2015-09-26 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: activesupport
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: unicode
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rake-compiler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.6'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.6'
+- !ruby/object:Gem::Dependency
+  name: minitest
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Implementation of Charikar simhashes in Ruby
+email:
+- alex.gusev@bookmate.ru
+- skapp@cortexinsight.com
+executables: []
+extensions:
+- ext/string_hashing/extconf.rb
+extra_rdoc_files: []
+files:
+- LICENSE
+- README.rdoc
+- Rakefile
+- ext/string_hashing/extconf.rb
+- ext/string_hashing/string_hashing.c
+- lib/integer.rb
+- lib/simhash.rb
+- lib/simhash/stopwords.rb
+- lib/simhash/stopwords/en.rb
+- lib/simhash/stopwords/ru.rb
+- lib/string.rb
+homepage: http://github.com/mort666/simhash
+licenses: []
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.3
+signing_key: 
+specification_version: 4
+summary: 'Gives you possbility to convert string into simhashes to futher use: finding
+  near-duplicates, similar strings, etc.'
+test_files: []