loose_tight_dictionary 0.0.10 → 0.1.0

data/loose_tight_dictionary.gemspec

@@ -1,71 +1,26 @@
-# Generated by jeweler
-# DO NOT EDIT THIS FILE DIRECTLY
-# Instead, edit Jeweler::Tasks in Rakefile, and run the gemspec command
 # -*- encoding: utf-8 -*-
+$:.push File.expand_path("../lib", __FILE__)
+require "loose_tight_dictionary/version"
 
 Gem::Specification.new do |s|
-  s.name = %q{loose_tight_dictionary}
-  s.version = "0.0.10"
+  s.name        = "loose_tight_dictionary"
+  s.version     = LooseTightDictionary::VERSION
+  s.platform    = Gem::Platform::RUBY
+  s.authors     = ["Seamus Abshere"]
+  s.email       = ["seamus@abshere.net"]
+  s.homepage    = "https://github.com/seamusabshere/loose_tight_dictionary"
+  s.summary     = %Q{Allows iterative development of dictionaries for big data sets.}
+  s.description = %Q{Create dictionaries that link rows between two tables using loose matching (string similarity) by default and tight matching (regexp) by request.}
 
-  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
-  s.authors = ["Seamus Abshere"]
-  s.date = %q{2011-03-02}
-  s.description = %q{Create dictionaries that link rows between two tables (left and right) using loose matching (string similarity) by default and tight matching (regexp) by request.}
-  s.email = %q{seamus@abshere.net}
-  s.extra_rdoc_files = [
-    "LICENSE",
-     "README.rdoc"
-  ]
-  s.files = [
-    ".document",
-     ".gitignore",
-     "LICENSE",
-     "README.rdoc",
-     "Rakefile",
-     "VERSION",
-     "examples/first_name_matching.rb",
-     "examples/icao-bts.rb",
-     "examples/icao-bts.xls",
-     "lib/loose_tight_dictionary.rb",
-     "loose_tight_dictionary.gemspec",
-     "test/helper.rb",
-     "test/test_loose_tight_dictionary.rb"
-  ]
-  s.homepage = %q{http://github.com/seamusabshere/loose_tight_dictionary}
-  s.rdoc_options = ["--charset=UTF-8"]
-  s.require_paths = ["lib"]
-  s.rubygems_version = %q{1.3.7}
-  s.summary = %q{Allows iterative development of dictionaries for big data sets.}
-  s.test_files = [
-    "test/helper.rb",
-     "test/test_loose_tight_dictionary.rb",
-     "examples/first_name_matching.rb",
-     "examples/icao-bts.rb"
-  ]
+  s.rubyforge_project = "loose_tight_dictionary"
 
-  if s.respond_to? :specification_version then
-    current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
-    s.specification_version = 3
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,floose_tight_dictionaryures}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ["lib"]
 
-    if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
-      s.add_development_dependency(%q<shoulda>, [">= 0"])
-      s.add_development_dependency(%q<remote_table>, [">= 0.2.19"])
-      s.add_runtime_dependency(%q<activesupport>, [">= 2.3.4"])
-      s.add_runtime_dependency(%q<andand>, [">= 1.3.1"])
-      s.add_runtime_dependency(%q<amatch>, [">= 0.2.5"])
-    else
-      s.add_dependency(%q<shoulda>, [">= 0"])
-      s.add_dependency(%q<remote_table>, [">= 0.2.19"])
-      s.add_dependency(%q<activesupport>, [">= 2.3.4"])
-      s.add_dependency(%q<andand>, [">= 1.3.1"])
-      s.add_dependency(%q<amatch>, [">= 0.2.5"])
-    end
-  else
-    s.add_dependency(%q<shoulda>, [">= 0"])
-    s.add_dependency(%q<remote_table>, [">= 0.2.19"])
-    s.add_dependency(%q<activesupport>, [">= 2.3.4"])
-    s.add_dependency(%q<andand>, [">= 1.3.1"])
-    s.add_dependency(%q<amatch>, [">= 0.2.5"])
-  end
+  s.add_development_dependency "shoulda"
+  s.add_development_dependency "remote_table"
+  s.add_dependency 'activesupport', '>=2.3.4'
+  s.add_dependency 'amatch'
 end
-

data/test/helper.rb

@@ -1,12 +1,12 @@
 require 'rubygems'
+require 'bundler'
+Bundler.setup
 require 'test/unit'
-require 'shoulda'
-require 'logger'
-# require 'ruby-debug'
-
-$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+require 'stringio'
+require 'remote_table'
 $LOAD_PATH.unshift(File.dirname(__FILE__))
-require File.expand_path(File.join(File.dirname(__FILE__), '..', 'lib', 'loose_tight_dictionary'))
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+require 'loose_tight_dictionary'
 
 class Test::Unit::TestCase
 end

data/test/test_blocking.rb

@@ -0,0 +1,23 @@
+require 'helper'
+
+class TestBlocking < Test::Unit::TestCase
+  def test_001_encompass_one
+    b = LooseTightDictionary::Blocking.new %r{apple}
+    assert_equal true, b.encompass?('2 apples')
+  end
+  
+  def test_002_encompass_both
+    b = LooseTightDictionary::Blocking.new %r{apple}
+    assert_equal true, b.encompass?('apple', '2 apples')    
+  end
+  
+  def test_002_doesnt_encompass_both
+    b = LooseTightDictionary::Blocking.new %r{apple}
+    assert_equal false, b.encompass?('orange', '2 apples')
+  end
+  
+  def test_003_no_information
+    b = LooseTightDictionary::Blocking.new %r{apple}
+    assert_equal nil, b.encompass?('orange', 'orange')
+  end
+end

data/test/test_extract_regexp.rb

@@ -0,0 +1,18 @@
+require 'helper'
+
+class TestExtractRegexp < Test::Unit::TestCase
+  def test_001_regexp
+    i = LooseTightDictionary::Identity.new %r{\A\\?/(.*)etc/mysql\$$}
+    assert_equal %r{\A\\?/(.*)etc/mysql\$$}, i.regexp
+  end
+  
+  def test_002_regexp_from_string
+    i = LooseTightDictionary::Identity.new '%r{\A\\\?/(.*)etc/mysql\$$}'
+    assert_equal %r{\A\\?/(.*)etc/mysql\$$}, i.regexp
+  end
+  
+  def test_003_regexp_from_string_using_slash_delim
+    i = LooseTightDictionary::Identity.new '/\A\\\?\/(.*)etc\/mysql\$$/'
+    assert_equal %r{\A\\?/(.*)etc/mysql\$$}, i.regexp
+  end
+end

data/test/test_identity.rb

@@ -0,0 +1,18 @@
+require 'helper'
+
+class TestIdentity < Test::Unit::TestCase
+  def test_001_identical
+    i = LooseTightDictionary::Identity.new %r{(A)[ ]*(\d)}
+    assert_equal true, i.identical?('A1', 'A     1foobar')
+  end
+  
+  def test_002_certainly_different
+    i = LooseTightDictionary::Identity.new %r{(A)[ ]*(\d)}
+    assert_equal false, i.identical?('A1', 'A     2foobar')    
+  end
+  
+  def test_003_no_information_ie_possible_identical
+    i = LooseTightDictionary::Identity.new %r{(A)[ ]*(\d)}
+    assert_equal nil, i.identical?('B1', 'A     2foobar')    
+  end
+end

data/test/test_loose_tight_dictionary.rb

@@ -1,271 +1,78 @@
 require 'helper'
 
-require 'remote_table'
-
-# $logger = Logger.new STDERR
-# $logger.level = Logger::DEBUG
-# $tee = STDOUT
-
 class TestLooseTightDictionary < Test::Unit::TestCase
-  def setup
-    clear_ltd
-    
-    # dh 8 400
-    @a_left = ['DE HAVILLAND CANADA DHC8400 Dash 8']
-    @a_right = ['DEHAVILLAND DEHAVILLAND DHC8-400 DASH-8']
-    # dh 88
-    @b_left = ['ABCDEFG DH88 HIJKLMNOP']
-    # dh 89
-    @c_right = ['ABCDEFG DH89 HIJKLMNOP']
-    # dh 8 200
-    @d_left = ['DE HAVILLAND CANADA DHC8200 Dash 8']
-    @d_right = ['BOMBARDIER DEHAVILLAND DHC8-200Q DASH-8']
-    @d_lookalike = ['ABCD DHC8200 Dash 8']
-    
-    @t_1 = [ '/(dh)c?-?(\d{0,2})-?(\d{0,4})(?:.*?)(dash|\z)/i', 'good tightening for de havilland' ]
-    
-    @r_1 = [ '/(dh)c?-?(\d{0,2})-?(\d{0,4})(?:.*?)(dash|\z)/i', 'good identity for de havilland' ]
-    
-    @left = [
-      @a_left,
-      @b_left,
-      ['DE HAVILLAND DH89 Dragon Rapide'],
-      ['DE HAVILLAND CANADA DHC8100 Dash 8 (E9, CT142, CC142)'],
-      @d_left,
-      ['DE HAVILLAND CANADA DHC8300 Dash 8'],
-      ['DE HAVILLAND DH90 Dragonfly']
-    ]
-    @right = [
-      @a_right,
-      @c_right,
-      @d_right,
-      ['DEHAVILLAND DEHAVILLAND DHC8-100 DASH-8'],
-      ['DEHAVILLAND DEHAVILLAND TWIN OTTER DHC-6']
-    ]
-    @tightenings = []
-    @identities = []
-    @blockings = []
-    @positives = []
-    @negatives = []
-  end
-
-  def clear_ltd
-    @_ltd = nil
-  end
-  
-  def ltd
-    @_ltd ||= LooseTightDictionary.new  @right,
-                                        :tightenings => @tightenings,
-                                        :identities => @identities,
-                                        :blockings => @blockings,
-                                        :positives => @positives,
-                                        :negatives => @negatives,
-                                        :blocking_only => @blocking_only,
-                                        :logger => $logger,
-                                        :tee => $tee
-  end
-
-  should "optionally only pay attention to things that match blockings" do
-    assert_equal @a_right, ltd.left_to_right(@a_left)
-
-    clear_ltd
-    @blocking_only = true
-    assert_equal nil, ltd.left_to_right(@a_left)
-
-    clear_ltd
-    @blocking_only = true
-    @blockings.push ['/dash/i']
-    assert_equal @a_right, ltd.left_to_right(@a_left)
-  end
+  # in case i start doing something with the log
+  # def setup
+  #   @log = StringIO.new
+  # end
+  # 
+  # def teardown
+  #   @log.close
+  # end
   
-  # the example from the readme, considerably uglier here
-  should "check a simple table" do
-    @right = [ 'seamus', 'andy', 'ben' ]
-    @positives = [ [ 'seamus', 'Mr. Seamus Abshere' ] ]
-    left = [ 'Mr. Seamus Abshere', 'Sr. Andy Rossmeissl', 'Master BenT' ]
-  
-    assert_nothing_raised do
-      ltd.check left
-    end
+  def test_001_find
+    d = LooseTightDictionary.new %w{ NISSAN HONDA }
+    assert_equal 'NISSAN', d.find('MISSAM')
   end
   
-  should "treat a String as a full record if passed through" do
-    dash = 'DHC8-400'
-    b747 = 'B747200/300'
-    dc9 = 'DC-9-10'
-    right_records = [ dash, b747, dc9 ]
-    simple_ltd = LooseTightDictionary.new right_records, :logger => $logger, :tee => $tee
-    assert_equal dash, simple_ltd.left_to_right('DeHavilland Dash-8 DHC-400')
-    assert_equal b747, simple_ltd.left_to_right('Boeing 747-300')
-    assert_equal dc9, simple_ltd.find('McDonnell Douglas MD81/DC-9')
+  def test_002_find_with_score
+    d = LooseTightDictionary.new %w{ NISSAN HONDA }
+    assert_equal ['NISSAN', 0.6], d.find_with_score('MISSAM')
   end
   
-  should "call it a mismatch if you hit a blank positive" do
-    @positives.push [@a_left[0], '']
-    assert_raises(LooseTightDictionary::Mismatch) do
-      ltd.left_to_right @a_left
-    end
-  end
-
-  should "call it a false positive if you hit a blank negative" do
-    @negatives.push [@a_left[0], '']
-    assert_raises(LooseTightDictionary::FalsePositive) do
-      ltd.left_to_right @a_left
-    end
+  def test_003_last_result
+    d = LooseTightDictionary.new %w{ NISSAN HONDA }
+    d.find 'MISSAM'
+    assert_equal 0.6, d.last_result.score
+    assert_equal 'NISSAN', d.last_result.record
   end
   
-  should "have a false match without blocking" do
-    # @d_left will be our victim
-    @right.push @d_lookalike
-    @tightenings.push @t_1
-    
-    assert_equal @d_lookalike, ltd.left_to_right(@d_left)
+  def test_004_false_positive_without_tightener
+    d = LooseTightDictionary.new ['BOEING 737-100/200', 'BOEING 737-900']
+    assert_equal 'BOEING 737-900', d.find('BOEING 737100 number 900')
   end
   
-  should "do blocking if the left matches a block" do
-    # @d_left will be our victim
-    @right.push @d_lookalike
-    @tightenings.push @t_1
-    @blockings.push ['/(bombardier|de ?havilland)/i']
-    
-    assert_equal @d_right, ltd.left_to_right(@d_left)
+  def test_005_correct_with_tightener
+    tighteners = [
+      %r{(7\d)(7|0)-?(\d{1,3})} # tighten 737-100/200 => 737100, which will cause it to win over 737-900
+    ]
+    d = LooseTightDictionary.new ['BOEING 737-100/200', 'BOEING 737-900'], :tighteners => tighteners
+    assert_equal 'BOEING 737-100/200', d.find('BOEING 737100 number 900')
   end
-  
-  should "treat blocks as exclusive" do
-    @right = [ @d_left ]
-    @tightenings.push @t_1
-    @blockings.push ['/(bombardier|de ?havilland)/i']
 
-    assert_equal nil, ltd.left_to_right(@d_lookalike)
-  end
-  
-  should "only use identities if they stem from the same regexp" do
-    @identities.push @r_1
-    @identities.push [ '/(cessna)(?:.*?)(citation)/i' ]
-    @identities.push [ '/(cessna)(?:.*?)(\d\d\d)/i' ]
-    x_left = [ 'CESSNA D-333 CITATION V']
-    x_right = [ 'CESSNA D-333' ]
-    @right.push x_right
-    
-    assert_equal x_right, ltd.left_to_right(x_left)
+  def test_008_false_positive_without_identity
+    d = LooseTightDictionary.new %w{ foo bar }
+    assert_equal 'bar', d.find('baz')
   end
   
-  should "use the best score from all of the tightenings" do
-    x_left = ["BOEING 737100"]
-    x_right = ["BOEING BOEING 737-100/200"]
-    x_right_wrong = ["BOEING BOEING 737-900"]
-    @right.push x_right
-    @right.push x_right_wrong
-    @tightenings.push ['/(7\d)(7|0)-?\d{1,3}\/(\d\d\d)/i']
-    @tightenings.push ['/(7\d)(7|0)-?(\d{1,3}|[A-Z]{0,3})/i']
-    
-    assert_equal x_right, ltd.left_to_right(x_left)
+  def test_008_identify_false_positive
+    d = LooseTightDictionary.new %w{ foo bar }, :identities => [ /ba(.)/ ]
+    assert_equal 'foo', d.find('baz')
   end
   
-  should "compare using prefixes if tightened key is shorter than correct match" do
-    x_left = ["BOEING 720"]
-    x_right = ["BOEING BOEING 720-000"]
-    x_right_wrong = ["BOEING BOEING 717-200"]
-    @right.push x_right
-    @right.push x_right_wrong
-    @tightenings.push @t_1
-    @tightenings.push ['/(7\d)(7|0)-?\d{1,3}\/(\d\d\d)/i']
-    @tightenings.push ['/(7\d)(7|0)-?(\d{1,3}|[A-Z]{0,3})/i']
+  def test_009_loose_blocking
+    # sanity check
+    d = LooseTightDictionary.new [ 'X' ]
+    assert_equal 'X', d.find('X')
+    assert_equal 'X', d.find('A')
+    # end sanity check
     
-    assert_equal x_right, ltd.left_to_right(x_left)
+    d = LooseTightDictionary.new [ 'X' ], :blockings => [ /X/, /Y/ ]
+    assert_equal 'X', d.find('X')
+    assert_equal 'X', d.find('A')
   end
   
-  should "use the shortest original input" do
-    x_left = ['De Havilland DHC8-777 Dash-8 Superstar']
-    x_right = ['DEHAVILLAND DEHAVILLAND DHC8-777 DASH-8 Superstar']
-    x_right_long = ['DEHAVILLAND DEHAVILLAND DHC8-777 DASH-8 Superstar/Supernova']
-    
-    @right.push x_right_long
-    @right.push x_right
-    @tightenings.push @t_1
-    
-    assert_equal x_right, ltd.left_to_right(x_left)
+  def test_010_strict_blocking
+    d = LooseTightDictionary.new [ 'X' ], :blockings => [ /X/, /Y/ ], :strict_blocking => true
+    assert_equal 'X', d.find('X')
+    assert_equal nil, d.find('A')
   end
   
-  should "perform lookups left to right" do
-    assert_equal @a_right, ltd.left_to_right(@a_left)
-  end
-
-  should "succeed if there are no checks" do
-    assert_nothing_raised do
-      ltd.check @left
-    end
-  end
-
-  should "succeed if the positive checks just work" do
-    @positives.push [ @a_left[0], @a_right[0] ]
-  
-    assert_nothing_raised do
-      ltd.check @left
-    end
-  end
-
-  should "fail if positive checks don't work" do
-    @positives.push [ @d_left[0], @d_right[0] ]
-
-    assert_raises(LooseTightDictionary::Mismatch) do
-      ltd.check @left
-    end
-  end
-
-  should "succeed if proper tightening is applied" do
-    @positives.push [ @d_left[0], @d_right[0] ]
-    @tightenings.push @t_1
-
-    assert_nothing_raised do
-      ltd.check @left
-    end
-  end
-
-  should "use a Google Docs spreadsheet as a source of tightenings" do
-    @positives.push [ @d_left[0], @d_right[0] ]
-    @tightenings = RemoteTable.new :url => 'http://spreadsheets.google.com/pub?key=tiS_6CCDDM_drNphpYwE_iw&single=true&gid=0&output=csv', :headers => false
-  
-    # sabshere 9/30/10 this shouldn't raise anything
-    # but the tightenings have been changed... we should be using test-only tightenings, not production ones
-    # assert_nothing_raised do
-    assert_raises(LooseTightDictionary::Mismatch) do
-      ltd.check @left
-    end
-  end
-  
-  should "fail if negative checks don't work" do
-    @negatives.push [ @b_left[0], @c_right[0] ]
-  
-    assert_raises(LooseTightDictionary::FalsePositive) do
-      ltd.check @left
-    end
-  end
-  
-  should "do inline checking" do
-    @negatives.push [ @b_left[0], @c_right[0] ]
-  
-    assert_raises(LooseTightDictionary::FalsePositive) do
-      ltd.left_to_right @b_left
-    end
-  end
-
-  should "fail if negative checks don't work, even with tightening" do
-    @negatives.push [ @b_left[0], @c_right[0] ]
-    @tightenings.push @t_1
-  
-    assert_raises(LooseTightDictionary::FalsePositive) do
-      ltd.check @left
-    end
-  end
-
-  should "succeed if proper identity is applied" do
-    @negatives.push [ @b_left[0], @c_right[0] ]
-    @positives.push [ @d_left[0], @d_right[0] ]
-    @identities.push @r_1
-  
-    assert_nothing_raised do
-      ltd.check @left
+  def test_011_free
+    d = LooseTightDictionary.new %w{ NISSAN HONDA }
+    d.free
+    assert_raises(LooseTightDictionary::Freed) do
+      d.find('foobar')
     end
   end
 end