loose_tight_dictionary 1.0.0 → 1.0.1

data/README.rdoc

@@ -17,11 +17,17 @@ Exclusively uses {Dice's Coefficient}[http://en.wikipedia.org/wiki/Dice's_coeffi
 
 Over 2 years in {Brighter Planet's environmental impact API}[http://impact.brighterplanet.com] and {reference data service}[http://data.brighterplanet.com].
 
-== Speed
+== Haystacks and how to read them
+
+The (admittedly imperfect) metaphor is "look for a needle in a haystack"
 
-If you add the amatch[http://flori.github.com/amatch/] gem to your Gemfile, it will use that, which is much faster (but {segfaults have been seen in the wild}[https://github.com/flori/amatch/issues/3]).
+* needle - the search term
+* haystack - the records you are searching (<b>your result will be an object from here</b>)
 
-Otherwise, a {pure ruby version}[http://stackoverflow.com/questions/653157/a-better-similarity-ranking-algorithm-for-variable-length-strings] is used.
+So, what if your needle is a string like <tt>youruguay</tt> and your haystack is full of <tt>Country</tt> objects like <tt><Country name:"Uruguay"></tt>?
+
+    >> LooseTightDictionary.new(countries, :read => :name).find('youruguay')
+    => <Country name:"Uruguay">
 
 == Regular expressions
 
@@ -49,7 +55,13 @@ Scoring is case-insensitive. Everything is downcased before scoring. This is a c
 
 == Examples
 
-Try running the included example files (<tt>examples/first_name_matching.rb</tt>) and check out the tests.
+Check out the tests.
+
+== Speed
+
+If you add the amatch[http://flori.github.com/amatch/] gem to your Gemfile, it will use that, which is much faster (but {segfaults have been seen in the wild}[https://github.com/flori/amatch/issues/3]). Thanks Flori!
+
+Otherwise, a pure ruby version derived from the {answer to a StackOverflow question}[http://stackoverflow.com/questions/653157/a-better-similarity-ranking-algorithm-for-variable-length-strings] is used. Thanks {marzagao}[http://stackoverflow.com/users/10997/marzagao]!
 
 == Authors
 

data/benchmark/memory.rb

@@ -37,7 +37,7 @@ TIGHTENERS = RemoteTable.new(:url => "file://#{File.expand_path("../../examples/
 IDENTITIES = RemoteTable.new(:url => "file://#{File.expand_path("../../examples/bts_aircraft/identities.csv", __FILE__)}", :headers => :first_row).map { |row| row['regexp'] }
 
 FINAL_OPTIONS = {
-  :haystack_reader => HAYSTACK_READER,
+  :read => HAYSTACK_READER,
   :must_match_blocking => MUST_MATCH_BLOCKING,
   :tighteners => TIGHTENERS,
   :identities => IDENTITIES,

data/examples/bts_aircraft/test_bts_aircraft.rb

@@ -63,7 +63,7 @@ NEGATIVES = RemoteTable.new :url => "file://#{File.expand_path("../negatives.csv
 # Section 3
 
 FINAL_OPTIONS = {
-  :haystack_reader => HAYSTACK_READER,
+  :read => HAYSTACK_READER,
   :must_match_blocking => MUST_MATCH_BLOCKING,
   :tighteners => TIGHTENERS,
   :identities => IDENTITIES,

data/lib/loose_tight_dictionary.rb

@@ -6,7 +6,7 @@ require 'active_support/version'
   active_support/core_ext/object
 }.each do |active_support_3_requirement|
   require active_support_3_requirement
-end if ::ActiveSupport::VERSION::MAJOR == 3
+end if ::ActiveSupport::VERSION::MAJOR >= 3
 require 'to_regexp'
 
 # See the README for more information.
@@ -20,24 +20,25 @@ class LooseTightDictionary
   autoload :Score, 'loose_tight_dictionary/score'
   autoload :CachedResult, 'loose_tight_dictionary/cached_result'
   
-  class Freed < RuntimeError; end
-  
   attr_reader :options
   attr_reader :haystack
   attr_reader :records
 
   # haystack - a bunch of records
   # options
-  # * tighteners: regexps that essentialize strings down
-  # * identities: regexps that rule out similarities, for example a 737 cannot be identical to a 747
+  # * tighteners: regexps (see readme)
+  # * identities: regexps
+  # * blockings: regexps
+  # * read: how to interpret each entry in the 'haystack', either a Proc or a symbol
   def initialize(records, options = {})
     @options = options.symbolize_keys
     @records = records
-    @haystack = records.map { |record| Wrapper.new :parent => self, :record => record, :reader => haystack_reader }
+    read = options[:read] || options[:haystack_reader]
+    @haystack = records.map { |record| Wrapper.new self, record, read }
   end
   
   def last_result
-    @last_result ||= Result.new
+    @last_result || raise(::RuntimeError, "[loose_tight_dictionary] You can't access the last result until you've run a find with :gather_last_result => true")
   end
   
   def log(str = '') #:nodoc:
@@ -50,11 +51,13 @@ class LooseTightDictionary
   end
   
   def find(needle, options = {})
-    raise Freed if freed?
-    free_last_result
+    raise ::RuntimeError, "[loose_tight_dictionary] Dictionary has already been freed, can't perform more finds" if freed?
     
     options = options.symbolize_keys
-    gather_last_result = options.fetch(:gather_last_result, true)
+    if gather_last_result = options.fetch(:gather_last_result, false)
+      free_last_result
+      @last_result = Result.new
+    end
     find_all = options.fetch(:find_all, false)
     
     if gather_last_result
@@ -63,7 +66,7 @@ class LooseTightDictionary
       last_result.blockings = blockings
     end
     
-    needle = Wrapper.new :parent => self, :record => needle
+    needle = Wrapper.new self, needle
     
     if gather_last_result
       last_result.needle = needle
@@ -126,7 +129,9 @@ class LooseTightDictionary
       last_result.similarities = similarities
     end
     
-    if best_similarity = similarities[-1] and straw = best_similarity.wrapper2 and record = straw.record
+    best_similarity = similarities[-1]
+    if best_similarity.best_score.to_f > 0
+      record = best_similarity.wrapper2.record
       if gather_last_result
         last_result.record = record
         last_result.score = best_similarity.best_score.to_f
@@ -140,7 +145,7 @@ class LooseTightDictionary
   #     d = LooseTightDictionary.new ['737', '747', '757' ]
   #     d.explain 'boeing 737-100'
   def explain(needle)
-    record = find needle
+    record = find needle, :gather_last_result => true
     log "#" * 150
     log "# Match #{needle.inspect} => #{record.inspect}"
     log "#" * 150
@@ -190,16 +195,12 @@ class LooseTightDictionary
     log record.inspect
   end
     
-  def haystack_reader
-    options[:haystack_reader]
-  end
-        
   def must_match_blocking
-    options[:must_match_blocking] || false
+    options.fetch :must_match_blocking, false
   end
   
   def first_blocking_decides
-    options[:first_blocking_decides] || false
+    options.fetch :first_blocking_decides, false
   end
 
   def tighteners

data/lib/loose_tight_dictionary/score.rb

@@ -43,6 +43,11 @@ class LooseTightDictionary
       def dices_coefficient(str1, str2)
         str1 = str1.downcase 
         str2 = str2.downcase
+        if str1 == str2
+          return 1.0
+        elsif str1.length == 1 and str2.length == 1
+          return 0.0
+        end
         pairs1 = (0..str1.length-2).map do |i|
           str1[i,2]
         end.reject do |pair|

data/lib/loose_tight_dictionary/version.rb

@@ -1,3 +1,3 @@
 class LooseTightDictionary
-  VERSION = '1.0.0'
+  VERSION = '1.0.1'
 end

data/lib/loose_tight_dictionary/wrapper.rb

@@ -3,12 +3,12 @@ class LooseTightDictionary
   class Wrapper #:nodoc: all
     attr_reader :parent
     attr_reader :record
-    attr_reader :reader
+    attr_reader :read
 
-    def initialize(attrs = {})
-      attrs.each do |k, v|
-        instance_variable_set "@#{k}", v
-      end
+    def initialize(parent, record, read = nil)
+      @parent = parent
+      @record = record
+      @read = read
     end
 
     def inspect
@@ -16,7 +16,20 @@ class LooseTightDictionary
     end
 
     def to_str
-      @to_str ||= reader ? reader.call(record) : record.to_s
+      @to_str ||= case read
+      when ::Proc
+        read.call record
+      when ::Symbol
+        if record.respond_to?(read)
+          record.send read
+        else
+          record[read]
+        end
+      when ::NilClass
+        record
+      else
+        record[read]
+      end.to_s
     end
 
     alias :to_s :to_str

data/test/test_cache.rb

@@ -33,7 +33,7 @@ class Aircraft < ActiveRecord::Base
   end
   
   def self.loose_tight_dictionary
-    @loose_tight_dictionary ||= LooseTightDictionary.new all, :haystack_reader => lambda { |straw| straw.aircraft_description }
+    @loose_tight_dictionary ||= LooseTightDictionary.new all, :read => ::Proc.new { |straw| straw.aircraft_description }
   end
   
   def self.create_table

data/test/test_loose_tight_dictionary.rb

@@ -13,17 +13,23 @@ class TestLooseTightDictionary < Test::Unit::TestCase
   def test_001_find
     d = LooseTightDictionary.new %w{ NISSAN HONDA }
     assert_equal 'NISSAN', d.find('MISSAM')
+    
+    d = LooseTightDictionary.new [ 'X' ]
+    assert_equal 'X', d.find('X')
+    assert_equal nil, d.find('A')
   end
   
-  def test_002_find_with_score
+  def test_002_dont_gather_last_result_by_default
     d = LooseTightDictionary.new %w{ NISSAN HONDA }
-    assert_equal 'NISSAN', d.find('MISSAM')
-    assert_equal 0.6, d.last_result.score
+    d.find('MISSAM')
+    assert_raises(::RuntimeError, /gather_last_result/) do
+      d.last_result
+    end
   end
   
   def test_003_last_result
     d = LooseTightDictionary.new %w{ NISSAN HONDA }
-    d.find 'MISSAM'
+    d.find 'MISSAM', :gather_last_result => true
     assert_equal 0.6, d.last_result.score
     assert_equal 'NISSAN', d.last_result.record
   end
@@ -48,18 +54,18 @@ class TestLooseTightDictionary < Test::Unit::TestCase
   
   def test_008_identify_false_positive
     d = LooseTightDictionary.new %w{ foo bar }, :identities => [ /ba(.)/ ]
-    assert_equal 'foo', d.find('baz')
+    assert_equal nil, d.find('baz')
   end
   
-  def test_009_must_match_blocking
-    d = LooseTightDictionary.new [ 'X' ]
-    assert_equal 'X', d.find('X')
-    assert_equal 'X', d.find('A')
-    
+  # TODO this is not very helpful
+  def test_009_blocking
     d = LooseTightDictionary.new [ 'X' ], :blockings => [ /X/, /Y/ ]
     assert_equal 'X', d.find('X')
-    assert_equal 'X', d.find('A')
-    
+    assert_equal nil, d.find('A')
+  end
+
+  # TODO this is not very helpful
+  def test_0095_must_match_blocking
     d = LooseTightDictionary.new [ 'X' ], :blockings => [ /X/, /Y/ ], :must_match_blocking => true
     assert_equal 'X', d.find('X')
     assert_equal nil, d.find('A')
@@ -68,7 +74,7 @@ class TestLooseTightDictionary < Test::Unit::TestCase
   def test_011_free
     d = LooseTightDictionary.new %w{ NISSAN HONDA }
     d.free
-    assert_raises(LooseTightDictionary::Freed) do
+    assert_raises(::RuntimeError, /free/) do
       d.find('foobar')
     end
   end
@@ -97,4 +103,54 @@ class TestLooseTightDictionary < Test::Unit::TestCase
     d = LooseTightDictionary.new [ 'Boeing 747', 'Boeing 747SR', 'Boeing ER6' ], :blockings => [ /(boeing \d{3})/i, /boeing/i ], :first_blocking_decides => true, :identities => [ /boeing (7|E)/i ]
     assert_equal [ 'Boeing ER6' ], d.find_all('Boeing ER6')
   end
+  
+  MyStruct = Struct.new(:one, :two)
+  def test_014_symbol_read_sends_method
+    ab = MyStruct.new('a', 'b')
+    ba = MyStruct.new('b', 'a')
+    haystack = [ab, ba]
+    by_first = LooseTightDictionary.new haystack, :read => :one
+    by_last = LooseTightDictionary.new haystack, :read => :two
+    assert_equal ab, by_first.find('a')
+    assert_equal ab, by_last.find('b')
+    assert_equal ba, by_first.find('b')
+    assert_equal ba, by_last.find('a')
+  end
+  
+  def test_015_symbol_read_reads_array
+    ab = ['a', 'b']
+    ba = ['b', 'a']
+    haystack = [ab, ba]
+    by_first = LooseTightDictionary.new haystack, :read => 0
+    by_last = LooseTightDictionary.new haystack, :read => 1
+    assert_equal ab, by_first.find('a')
+    assert_equal ab, by_last.find('b')
+    assert_equal ba, by_first.find('b')
+    assert_equal ba, by_last.find('a')
+  end
+  
+  def test_016_symbol_read_reads_hash
+    ab = { :one => 'a', :two => 'b' }
+    ba = { :one => 'b', :two => 'a' }
+    haystack = [ab, ba]
+    by_first = LooseTightDictionary.new haystack, :read => :one
+    by_last = LooseTightDictionary.new haystack, :read => :two
+    assert_equal ab, by_first.find('a')
+    assert_equal ab, by_last.find('b')
+    assert_equal ba, by_first.find('b')
+    assert_equal ba, by_last.find('a')
+  end
+  
+  def test_017_understands_haystack_reader_option
+    ab = ['a', 'b']
+    ba = ['b', 'a']
+    haystack = [ab, ba]
+    by_first = LooseTightDictionary.new haystack, :haystack_reader => 0
+    assert_equal ab, by_first.find('a')
+    assert_equal ba, by_first.find('b')
+  end
+  
+  def test_018_no_result_if_best_score_is_zero
+    assert_equal nil, LooseTightDictionary.new(['a']).find('b')
+  end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: loose_tight_dictionary
 version: !ruby/object:Gem::Version
-  version: 1.0.0
+  version: 1.0.1
   prerelease: 
 platform: ruby
 authors:
@@ -13,7 +13,7 @@ date: 2011-12-03 00:00:00.000000000Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: shoulda
-  requirement: &2160898300 !ruby/object:Gem::Requirement
+  requirement: &2164915940 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -21,10 +21,10 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *2160898300
+  version_requirements: *2164915940
 - !ruby/object:Gem::Dependency
   name: remote_table
-  requirement: &2160965820 !ruby/object:Gem::Requirement
+  requirement: &2164983440 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -32,10 +32,10 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *2160965820
+  version_requirements: *2164983440
 - !ruby/object:Gem::Dependency
   name: activerecord
-  requirement: &2161059600 !ruby/object:Gem::Requirement
+  requirement: &2165077220 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -43,10 +43,10 @@ dependencies:
         version: '3'
   type: :development
   prerelease: false
-  version_requirements: *2161059600
+  version_requirements: *2165077220
 - !ruby/object:Gem::Dependency
   name: mysql
-  requirement: &2161138940 !ruby/object:Gem::Requirement
+  requirement: &2165156580 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -54,10 +54,10 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *2161138940
+  version_requirements: *2165156580
 - !ruby/object:Gem::Dependency
   name: cohort_scope
-  requirement: &2161208000 !ruby/object:Gem::Requirement
+  requirement: &2165225600 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -65,10 +65,10 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *2161208000
+  version_requirements: *2165225600
 - !ruby/object:Gem::Dependency
   name: weighted_average
-  requirement: &2161382600 !ruby/object:Gem::Requirement
+  requirement: &2165400220 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -76,10 +76,10 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *2161382600
+  version_requirements: *2165400220
 - !ruby/object:Gem::Dependency
   name: rake
-  requirement: &2161619140 !ruby/object:Gem::Requirement
+  requirement: &2165637020 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -87,10 +87,10 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *2161619140
+  version_requirements: *2165637020
 - !ruby/object:Gem::Dependency
   name: activesupport
-  requirement: &2161769980 !ruby/object:Gem::Requirement
+  requirement: &2165774580 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -98,10 +98,10 @@ dependencies:
         version: '3'
   type: :runtime
   prerelease: false
-  version_requirements: *2161769980
+  version_requirements: *2165774580
 - !ruby/object:Gem::Dependency
   name: to_regexp
-  requirement: &2161790420 !ruby/object:Gem::Requirement
+  requirement: &2165808840 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -109,7 +109,7 @@ dependencies:
         version: 0.0.3
   type: :runtime
   prerelease: false
-  version_requirements: *2161790420
+  version_requirements: *2165808840
 description: Create dictionaries that link rows between two tables using loose matching
   (string similarity) by default and tight matching (regexp) by request.
 email: