simhilarity 1.0.0 → 1.0.1

data/lib/simhilarity/bits.rb

@@ -26,37 +26,5 @@ module Simhilarity
       b = (x >>  0) & 0xffff
       HAMMING16[a] + HAMMING16[b]
     end
-
-    # can't rely on ruby hash, because it's not consistent across
-    # sessions. Let's just use MD5.
-    def self.nhash(ngram)
-      @hashes ||= { }
-      @hashes[ngram] ||= Digest::MD5.hexdigest(ngram).to_i(16)
-    end
-
-    # Calculate the frequency weighted
-    # simhash[http://matpalm.com/resemblance/simhash/] of the
-    # +ngrams+.
-    def self.simhash32(freq, ngrams)
-      # array of bit sums
-      bits = Array.new(32, 0)
-
-      # walk bits of ngram's hash, increase/decrease bit sums
-      ngrams.each do |ngram|
-        f = freq[ngram]
-        hash = nhash(ngram)
-        (0...32).each do |i|
-          bits[i] += (((hash >> i) & 1) == 1) ? f : -f
-        end
-      end
-
-      # calculate simhash based on whether bit sums are negative or
-      # positive
-      simhash = 0
-      (0...32).each do |bit|
-        simhash |= (1 << bit) if bits[bit] > 0
-      end
-      simhash
-    end
   end
 end

data/lib/simhilarity/matcher.rb

@@ -37,7 +37,7 @@ module Simhilarity
       self.normalizer = options[:normalizer]
       self.ngrammer = options[:ngrammer]
 
-      self.freq = Hash.new(1)
+      reset_corpus
     end
 
     # Set the corpus. Calculates ngram frequencies (#freq) for future
@@ -45,6 +45,8 @@ module Simhilarity
     def corpus=(corpus)
       @corpus = corpus
 
+      reset_corpus
+
       # calculate ngram counts for the corpus
       counts = Hash.new(0)
       veach("Corpus", import_list(corpus)) do |element|
@@ -54,10 +56,9 @@ module Simhilarity
       end
 
       # turn counts into inverse frequencies
-      self.freq = Hash.new(1)
       total = counts.values.inject(&:+).to_f
       counts.each do |ngram, count|
-        self.freq[ngram] = total / count
+        @freq[ngram] = ((total / count) * 10).round
       end
     end
 
@@ -114,7 +115,16 @@ module Simhilarity
     # simhash[http://matpalm.com/resemblance/simhash/] of the
     # +ngrams+.
     def simhash(ngrams)
-      Bits.simhash32(freq, ngrams)
+      # map each ngram to its bitsums
+      sums = ngrams.map { |i| simhash_bitsums(i) }
+      # transpose and calculate final sum for each bit
+      bits = sums.transpose.map { |values| values.inject(&:+) }
+      # wherever we have a positive sum, the simhash bit is 1
+      simhash = 0
+      bits.each_with_index do |i, index|
+        simhash |= (1 << index) if i > 0
+      end
+      simhash
     end
 
     def inspect #:nodoc:
@@ -137,6 +147,30 @@ module Simhilarity
       Element.new(self, opaque)
     end
 
+    def reset_corpus
+      @freq = Hash.new(1)
+      @bitsums = { }
+    end
+
+    # calculate the simhash bitsums for this +ngram+, as part of
+    # calculating the simhash. We can cache this because it only
+    # depends on the freq and ngram.
+    def simhash_bitsums(ngram)
+      @bitsums[ngram] ||= begin
+        # hash the ngram using a consistent hash (ruby's hash changes
+        # across sessions)
+        hash = Digest::MD5.hexdigest(ngram).to_i(16)
+
+        # map hash bits, 1 ? f : -f
+        f = freq[ngram]
+        array = Array.new(32, 0)
+        (0...32).each do |i|
+          array[i] = (((hash >> i) & 1) == 1) ? f : -f
+        end
+        array
+      end
+    end
+
     # Puts if options[:verbose]
     def vputs(s)
       $stderr.puts s if options[:verbose]

data/lib/simhilarity/version.rb

@@ -1,4 +1,4 @@
 module Simhilarity
   # Gem version
-  VERSION = "1.0.0"
+  VERSION = "1.0.1"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: simhilarity
 version: !ruby/object:Gem::Version
-  version: 1.0.0
+  version: 1.0.1
   prerelease: 
 platform: ruby
 authors:
@@ -140,7 +140,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: -1497809244519171705
+      hash: -2275181007329764325
 requirements: []
 rubyforge_project: simhilarity
 rubygems_version: 1.8.24