simhilarity 1.0.0

data/lib/simhilarity/matcher.rb

@@ -0,0 +1,164 @@
+require "progressbar"
+
+module Simhilarity
+  # Abstract superclass for matching. Mainly a container for options, corpus, etc.
+  class Matcher
+    # Options used to create this Matcher.
+    attr_accessor :options
+
+    # Proc for turning needle/haystack elements into strings. You can
+    # leave this nil if the elements are already strings. See
+    # Matcher#reader for the default implementation.
+    attr_accessor :reader
+
+    # Proc for normalizing input strings. See Matcher#normalize
+    # for the default implementation.
+    attr_accessor :normalizer
+
+    # Proc for generating ngrams from a normalized string. See
+    # Matcher#ngrams for the default implementation.
+    attr_accessor :ngrammer
+
+    # Ngram frequency weights from the corpus, or 1 if the ngram isn't
+    # in the corpus.
+    attr_accessor :freq
+
+    # Create a new Matcher matcher. Options include:
+    #
+    # * +reader+: Proc for turning opaque items into strings.
+    # * +normalizer+: Proc for normalizing strings.
+    # * +ngrammer+: Proc for generating ngrams.
+    # * +verbose+: If true, show progress bars and timing.
+    def initialize(options = {})
+      @options = options
+
+      # procs
+      self.reader = options[:reader]
+      self.normalizer = options[:normalizer]
+      self.ngrammer = options[:ngrammer]
+
+      self.freq = Hash.new(1)
+    end
+
+    # Set the corpus. Calculates ngram frequencies (#freq) for future
+    # scoring.
+    def corpus=(corpus)
+      @corpus = corpus
+
+      # calculate ngram counts for the corpus
+      counts = Hash.new(0)
+      veach("Corpus", import_list(corpus)) do |element|
+        element.ngrams.each do |ngram|
+          counts[ngram] += 1
+        end
+      end
+
+      # turn counts into inverse frequencies
+      self.freq = Hash.new(1)
+      total = counts.values.inject(&:+).to_f
+      counts.each do |ngram, count|
+        self.freq[ngram] = total / count
+      end
+    end
+
+    # The current corpus.
+    def corpus
+      @corpus
+    end
+
+    # Turn an opaque item from the user into a string.
+    def read(opaque)
+      if reader
+        return reader.call(opaque)
+      end
+
+      if opaque.is_a?(String)
+        return opaque
+      end
+      raise "can't turn #{opaque.inspect} into string"
+    end
+
+    # Normalize an incoming string from the user.
+    def normalize(incoming_str)
+      if normalizer
+        return normalizer.call(incoming_str)
+      end
+
+      str = incoming_str
+      str = str.downcase
+      str = str.gsub(/[^a-z0-9]/, " ")
+      # squish whitespace
+      str = str.gsub(/\s+/, " ").strip
+      str
+    end
+
+    # Generate ngrams from a normalized str.
+    def ngrams(str)
+      if ngrammer
+        return ngrammer.call(str)
+      end
+
+      # two letter ngrams (bigrams)
+      ngrams = str.each_char.each_cons(2).map(&:join)
+      # runs of digits
+      ngrams += str.scan(/\d+/)
+      ngrams.uniq
+    end
+
+    # Sum up the frequency weights of the +ngrams+.
+    def ngrams_sum(ngrams)
+      ngrams.map { |i| @freq[i] }.inject(&:+) || 0
+    end
+
+    # Calculate the frequency weighted
+    # simhash[http://matpalm.com/resemblance/simhash/] of the
+    # +ngrams+.
+    def simhash(ngrams)
+      Bits.simhash32(freq, ngrams)
+    end
+
+    def inspect #:nodoc:
+      "Matcher"
+    end
+
+    protected
+
+    # Turn a list of user supplied opaque items into a list of
+    # Elements (if necessary).
+    def import_list(list)
+      if !list.first.is_a?(Element)
+        list = list.map { |opaque| element_for(opaque) }
+      end
+      list
+    end
+
+    # Turn a user's opaque item into an Element.
+    def element_for(opaque)
+      Element.new(self, opaque)
+    end
+
+    # Puts if options[:verbose]
+    def vputs(s)
+      $stderr.puts s if options[:verbose]
+    end
+
+    # Like each, but with a progress bar if options[:verbose]
+    def veach(title, array, &block)
+      if !options[:verbose]
+        array.each do |i|
+          yield(i)
+        end
+      else
+        begin
+          pb = ProgressBar.new(title, array.length)
+          array.each do |i|
+            yield(i)
+            pb.inc
+          end
+        ensure
+          pb.finish
+        end
+      end
+    end
+  end
+end

data/lib/simhilarity/single.rb

@@ -0,0 +1,18 @@
+require "set"
+
+module Simhilarity
+  # Calculate the similarity score for pairs of items, one at a time.
+  class Single < Matcher
+    # See Matcher#initialize.
+    def initialize(options = {})
+      super(options)
+    end
+
+    # Calculate the similarity score for these two items. Scores range
+    # from 0 to 1, with 1 being a perfect match and 0 being a terrible
+    # match. For best results, call #corpus= first.
+    def score(a, b)
+      Candidate.new(self, element_for(a), element_for(b)).score
+    end
+  end
+end

data/lib/simhilarity/version.rb

@@ -0,0 +1,4 @@
+module Simhilarity
+  # Gem version
+  VERSION = "1.0.0"
+end

data/lib/simhilarity.rb

@@ -0,0 +1,8 @@
+require "simhilarity/bits"
+require "simhilarity/candidate"
+require "simhilarity/element"
+require "simhilarity/matcher"
+require "simhilarity/version"
+
+require "simhilarity/bulk"
+require "simhilarity/single"

data/simhilarity.gemspec

@@ -0,0 +1,27 @@
+require "simhilarity/version"
+
+Gem::Specification.new do |s|
+  s.name        = "simhilarity"
+  s.version     = Simhilarity::VERSION
+  s.platform    = Gem::Platform::RUBY
+  s.required_ruby_version = ">= 1.9.0"
+  s.authors     = ["Adam Doppelt"]
+  s.email       = ["amd@gurge.com"]
+  s.homepage    = "http://github.com/gurgeous/simhilarity"
+  s.summary     = "Simhilarity - measure text similarity using frequency weighted ngrams."
+  s.description = "Measure text similarity using frequency weighted ngrams."
+
+  s.rubyforge_project = "simhilarity"
+
+  s.add_runtime_dependency "bk"
+  s.add_runtime_dependency "progressbar"
+
+  s.add_development_dependency "awesome_print"
+  s.add_development_dependency "rake"
+  s.add_development_dependency "rdoc"
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- test/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map { |i| File.basename(i) }
+  s.require_paths = ["lib"]
+end

data/test/harness

@@ -0,0 +1,138 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+
+require "awesome_print"
+require "csv"
+require "ostruct"
+require "simhilarity"
+
+#
+# This is a test harness for measuring the speed and accuracy of the
+# different searchers. Feel free to ignore.
+#
+
+#
+# Results on an i5 3ghz, with 500 needles and 10,000 haystacks:
+#
+#
+
+class Harness
+  def initialize
+    @matcher = Simhilarity::Bulk.new
+
+    # load data
+    $stderr.puts "Loading..."
+    @data = dataset_large
+    @data.needles  = @data.needles.map  { |i| @matcher.send(:element_for, i) }
+    @data.haystack = @data.haystack.map { |i| @matcher.send(:element_for, i) }
+    @matcher.corpus = @data.needles + @data.haystack
+
+    # exclude this initialization from our benchmarks
+    @matcher.corpus.each(&:simhash)
+
+    $stderr.puts "Harness using needles/haystack = #{@data.needles.length}/#{@data.haystack.length}..."
+    run
+  end
+
+  def dataset_large
+    needles  = File.readlines("large_needles.txt").map(&:chomp)
+    haystack = File.readlines("large_haystack.txt").map(&:chomp)
+    OpenStruct.new(needles: needles, haystack: haystack)
+  end
+
+  def dataset_small
+    needles, haystack, matches = [], [], {}
+    CSV.read("sample.csv").each do |cols|
+      n, h = *cols
+      needles << n if n
+      haystack << h if h
+      matches[n] = h if n && h
+    end
+    OpenStruct.new(needles: needles, haystack: haystack, matches: matches)
+  end
+
+  # all
+  # ngrams:  5,4,3
+  # simhash: 5,6,7,8,9
+
+  #
+  # compare the various candidate methods
+  #
+
+  def run
+    # header
+    cols = [:title, :candidates, :s30, :s40, :s50, :c_tm, :s_tm, :correct]
+    puts cols.join("\t")
+
+    # simhash: 5..9
+    @matcher.options[:candidates] = :simhash
+    5.upto(9).each do |i|
+      @matcher.options[:simhash_max_hamming] = i
+      report("simhash #{i}")
+    end
+
+    # ngrams: 5..3
+    @matcher.options[:candidates] = :ngrams
+    5.downto(3).each do |i|
+      @matcher.options[:ngram_overlaps] = i
+      report("ngrams #{i}")
+    end
+
+    # all
+    @matcher.options[:candidates] = :all
+    report("all")
+  end
+
+
+  def report(title)
+    # candidates
+    tm1 = Time.now
+    candidates = @matcher.send(:candidates, @data.needles, @data.haystack)
+    tm1 = Time.now - tm1
+
+    # winners
+    tm2 = Time.now
+    winners = @matcher.send(:winners, @data.needles, candidates)
+    tm2 = Time.now - tm2
+    winners = winners.sort_by { |n, h, score| -(score || 0) }
+
+    s30 = winners.count { |n, h, score| score && score > 0.3 }
+    s40 = winners.count { |n, h, score| score && score > 0.4 }
+    s50 = winners.count { |n, h, score| score && score > 0.5 }
+
+    tm1 = sprintf("%.3f", tm1)
+    tm2 = sprintf("%.3f", tm2)
+
+    cols = [title, candidates.length, s30, s40, s50, tm1, tm2]
+    if @data.matches
+      correct = winners.select { |n, h, score| @data.matches[n] == h }
+      correct = correct.length.to_f / @data.needles.length
+      correct = sprintf("%.3f", correct)
+      cols << correct
+    end
+    puts cols.join("\t")
+    $stdout.flush
+  end
+
+  def dump_results(winners)
+    high_quality = winners.select { |n, h, score| score && score > 0.5 }
+    full report
+    high_quality.each do |n, h, score|
+      printf("%4.2f %-35s %-35s\n", score || 0, n, h)
+    end
+
+    # which high quality matches were added?
+    if @last
+      puts
+      added = high_quality - @last
+      added.each do |n, h, score|
+        printf("%4.2f %-35s %-35s\n", score || 0, n, h)
+      end
+    end
+    @last = high_quality
+  end
+end
+
+Dir.chdir(File.expand_path("../", __FILE__))
+Harness.new

data/test/identity.txt

@@ -0,0 +1 @@
+hello, world!