simhilarity 1.0.0

data/test/tests.rb

@@ -0,0 +1,125 @@
+require "awesome_print"
+require "benchmark"
+require "csv"
+require "ostruct"
+require "simhilarity"
+require "test/unit"
+
+class Tests < Test::Unit::TestCase
+  #
+  # helpers
+  #
+
+  def setup
+    Dir.chdir(File.expand_path("../", __FILE__))
+    @matcher = Simhilarity::Matcher.new
+  end
+
+  # Read the sample.csv test data file. needle is a list of needle
+  # strings, haystack is a list of haystack strings, and matches is a
+  # hash mapping from needle to haystack for known good matches.
+  def sample
+    sample = OpenStruct.new(needle: [], haystack: [], matches: { })
+    CSV.read("sample.csv").each do |cols|
+      n, h = *cols
+      sample.needle << n if n
+      sample.haystack << h if h
+      sample.matches[n] = h if n && h
+    end
+    sample
+  end
+
+  def assert_bulk_candidates(candidates, percent)
+    sample = self.sample
+
+    # match, with benchmark
+    output = nil
+    Benchmark.bm(10) do |bm|
+      bm.report(candidates.to_s) do
+        matcher = Simhilarity::Bulk.new(candidates: candidates)
+        output = matcher.matches(sample.needle, sample.haystack)
+      end
+    end
+
+    # what percent of matches are correct?
+    correct = output.select { |n, h, score| sample.matches[n] == h }
+    correct = correct.length.to_f / sample.needle.length
+
+    # for debugging
+    # printf("%% correct: %.3f\n", correct)
+    # output.each do |n, h, score|
+    #   good = sample.matches[n] == h
+    #   printf("%2s %4.2f %-35s %-35s\n", good ? "" : "xx", score || 0, n, h)
+    # end
+
+    assert((correct - percent).abs < 0.001, "percent #{correct} != #{percent}")
+  end
+
+  def assert_system(cmd)
+    system("#{cmd} > /dev/null 2>&1")
+    assert($? == 0, "#{cmd} failed")
+  end
+
+  #
+  # tests
+  #
+
+  def test_read
+    # default
+    assert_equal @matcher.read("gub"), "gub"
+
+    # not a string
+    assert_raise(RuntimeError) { @matcher.read(123) }
+
+    # custom
+    @matcher.reader = lambda(&:key)
+    assert_equal @matcher.read(OpenStruct.new(key: "gub")), "gub"
+  end
+
+  def test_normalizer
+    # default
+    assert_equal @matcher.normalize(" HELLO,\tWORLD! "), "hello world"
+
+    # custom
+    @matcher.normalizer = lambda(&:upcase)
+    assert_equal @matcher.normalize("gub"), "GUB"
+  end
+
+  def test_ngrams
+    # default
+    assert_equal @matcher.ngrams("hi 42"), ["hi", "i ", " 4", "42"]
+
+    # custom
+    @matcher.ngrammer = lambda(&:split)
+    assert_equal @matcher.ngrams("hi 42"), ["hi", "42"]
+  end
+
+  def test_proc_options
+    matcher = Simhilarity::Matcher.new(reader: lambda(&:key), normalizer: lambda(&:upcase), ngrammer: lambda(&:split))
+    assert_equal matcher.read(OpenStruct.new(key: "gub")), "gub"
+    assert_equal matcher.normalize("gub"), "GUB"
+    assert_equal matcher.ngrams("hi 42"), ["hi", "42"]
+  end
+
+  def test_single
+    score = Simhilarity::Single.new.score("hello world", "hi worlds")
+    assert (score - 0.556).abs < 0.001, "test_single percent was wrong!"
+  end
+
+  def test_bulk
+    assert_bulk_candidates(:all, 0.974)
+    assert_bulk_candidates(:ngrams, 0.974)
+    assert_bulk_candidates(:simhash, 0.949)
+  end
+
+  def test_bin
+    bin = "../bin/simhilarity"
+    assert_system("#{bin} identity.txt identity.txt")
+    assert_system("#{bin} -v identity.txt identity.txt")
+    assert_system("#{bin} --candidates simhash identity.txt identity.txt")
+    assert_system("#{bin} --candidates simhash=3 identity.txt identity.txt")
+    assert_system("#{bin} --candidates ngrams identity.txt identity.txt")
+    assert_system("#{bin} --candidates ngrams=3 identity.txt identity.txt")
+    assert_system("#{bin} --candidates all identity.txt identity.txt")
+  end
+end

metadata

@@ -0,0 +1,156 @@
+--- !ruby/object:Gem::Specification
+name: simhilarity
+version: !ruby/object:Gem::Version
+  version: 1.0.0
+  prerelease: 
+platform: ruby
+authors:
+- Adam Doppelt
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-04-18 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bk
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: progressbar
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: awesome_print
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rdoc
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Measure text similarity using frequency weighted ngrams.
+email:
+- amd@gurge.com
+executables:
+- simhilarity
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- LICENSE
+- README.md
+- Rakefile
+- bin/simhilarity
+- lib/simhilarity.rb
+- lib/simhilarity/bits.rb
+- lib/simhilarity/bulk.rb
+- lib/simhilarity/candidate.rb
+- lib/simhilarity/element.rb
+- lib/simhilarity/matcher.rb
+- lib/simhilarity/single.rb
+- lib/simhilarity/version.rb
+- simhilarity.gemspec
+- test/harness
+- test/identity.txt
+- test/large_haystack.txt
+- test/large_needles.txt
+- test/sample.csv
+- test/tests.rb
+homepage: http://github.com/gurgeous/simhilarity
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: 1.9.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+      segments:
+      - 0
+      hash: -1497809244519171705
+requirements: []
+rubyforge_project: simhilarity
+rubygems_version: 1.8.24
+signing_key: 
+specification_version: 3
+summary: Simhilarity - measure text similarity using frequency weighted ngrams.
+test_files:
+- test/harness
+- test/identity.txt
+- test/large_haystack.txt
+- test/large_needles.txt
+- test/sample.csv
+- test/tests.rb