simhash 0.0.1

data/LICENSE

@@ -0,0 +1,28 @@
+
+LICENSE
+
+The MIT License
+
+Copyright (c) 2002 Charikar, Simhash algorythm
+Copyright (c) 2009 Andre Hagenbruch, Python implementation
+Copyright (c) 2010 Bookmate.ru
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+
+

data/README

@@ -0,0 +1 @@
+Implementation of Charikar simhashes in Ruby

data/Rakefile

@@ -0,0 +1,46 @@
+require 'rake'
+require 'rake/testtask'
+require 'rake/rdoctask'
+
+$LOAD_PATH << File.join(File.dirname(__FILE__), 'lib')
+require 'simhash'
+
+desc 'Default: run unit tests.'
+task :default => [:test]
+
+desc 'Test the simhash gem'
+Rake::TestTask.new(:test) do |t|
+  t.libs << 'lib'
+  t.pattern = 'test/**/*_test.rb'
+  t.verbose = true
+end
+
+desc 'Start an IRB session with all necessary files required.'
+task :shell do |t|
+  chdir File.dirname(__FILE__)
+  exec 'irb -I lib/ -I lib/simhash -I lib/string -I lib/integer -r rubygems -r init'
+end
+
+desc 'Build the gemspec.'
+task :gemspec do |t|
+  exec 'gem build simhash.gemspec'
+end
+
+desc "Print a list of the files to be put into the gem"
+task :manifest do
+  spec.files.each do |file|
+    puts file
+  end
+end
+
+desc "Generate a gemspec file for GitHub"
+task :gemspec do
+  File.open("#{spec.name}.gemspec", 'w') do |f|
+    f.write spec.to_ruby
+  end
+end
+
+desc "Build the gem into the current directory"
+task :gem => :gemspec do
+  `gem build #{spec.name}.gemspec`
+end

data/init.rb

@@ -0,0 +1,3 @@
+require File.join(File.dirname(__FILE__), "lib", "simhash")
+require File.join(File.dirname(__FILE__), "lib", "string")
+require File.join(File.dirname(__FILE__), "lib", "integer")

data/lib/integer.rb

@@ -0,0 +1,16 @@
+class Integer
+  # Hamming distance – number of different bits in same positions
+  # H(1001, 1110) = 3 
+  # H(1001, 1000) = 1
+  def hamming_distance_to(integer)
+    total = 0
+    difference = self ^ integer
+
+    while difference > 0 do
+      total += 1 if (difference & 1).nonzero?
+      difference >>= 1
+    end
+    
+    total
+  end
+end

data/lib/simhash.rb

@@ -0,0 +1,56 @@
+class Simhash
+  attr_accessor :hashbits, :hash
+  def initialize(tokens='', hashbits=128)
+    self.hashbits = hashbits
+    self.hash = self.simhash(tokens)
+    hash
+  end
+  
+  def to_s
+    self.hash.to_s
+  end
+  
+  def to_i
+    self.hash.to_i
+  end
+  
+  def simhash(tokens)
+    v = [0] * self.hashbits
+    masks = v.dup
+    masks.each_with_index {|e, i| masks[i] = (1 << i)}
+    tokens.each do |token|
+      hashed_token = token.hash_wl(self.hashbits)
+      bitmask = 0
+      self.hashbits.times do |i|
+        v[i] += (hashed_token & masks[i]).zero? ? -1 : +1
+      end
+    end
+   
+    fingerprint = 0
+
+    self.hashbits.times { |i| fingerprint += 1 << i if v[i] >= 0 }  
+      
+    fingerprint    
+  end
+  
+  def hamming_distance(other_hash)
+    self.hash.hamming_distance_to(other_hash.hash)
+  end
+
+  def string_hash(string)
+    if string == ""
+      return 0
+    else
+      x = string[0] << 7
+      m = 1000003
+      mask = (1<<self.hashbits) - 1
+      string.each_byte do |char| 
+        x = ((x * m) ^ char) & mask
+      end
+          
+      x ^= string.size
+      x = -2 if x == -1
+      x
+    end
+  end
+end

data/lib/string.rb

@@ -0,0 +1,19 @@
+class String
+  def simhash(hashbits=128)
+    Simhash.new(self.split, hashbits)
+  end
+  
+  # string hash of predefined length
+  def hash_wl(length)
+    return 0 if self == ""
+
+    x = self[0] << 7
+    m = 1000003
+    mask = (1<<length) - 1
+    self.each_byte{ |char| x = ((x * m) ^ char) & mask }
+        
+    x ^= self.size
+    x = -2 if x == -1
+    x
+  end
+end

data/rails/init.rb

@@ -0,0 +1,3 @@
+require File.join(File.dirname(__FILE__), "..", "lib", "simhash")
+require File.join(File.dirname(__FILE__), "..", "lib", "string")
+require File.join(File.dirname(__FILE__), "..", "lib", "integer")

metadata

@@ -0,0 +1,74 @@
+--- !ruby/object:Gem::Specification 
+name: simhash
+version: !ruby/object:Gem::Version 
+  hash: 29
+  prerelease: false
+  segments: 
+  - 0
+  - 0
+  - 1
+  version: 0.0.1
+platform: ruby
+authors: 
+- Alex Gusev
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2010-08-11 00:00:00 +04:00
+default_executable: 
+dependencies: []
+
+description: Implementation of Charikar simhashes in Ruby
+email: alex.gusev@bookmate.ru
+executables: []
+
+extensions: []
+
+extra_rdoc_files: []
+
+files: 
+- README
+- LICENSE
+- Rakefile
+- init.rb
+- lib/integer.rb
+- lib/simhash.rb
+- lib/string.rb
+- rails/init.rb
+has_rdoc: true
+homepage: http://github.com/bookmate/simhash
+licenses: []
+
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+requirements: []
+
+rubyforge_project: simhash
+rubygems_version: 1.3.7
+signing_key: 
+specification_version: 3
+summary: "Gives you possbility to convert string into simhashes to futher use: finding near-duplicates, similar strings, etc."
+test_files: []
+