loose_tight_dictionary 0.1.1 → 0.2.0

data/benchmark/memory.rb

@@ -22,7 +22,7 @@ HAYSTACK_READER = lambda { |record| "#{record['Manufacturer']} #{record['Long Na
 
 # Whether to even bother trying to find a match for something without an explicit block
 # (Example) False, which is the default, which means we have more work to do
-STRICT_BLOCKING = false
+MUST_MATCH_BLOCKING = false
 
 # Blockings
 # (Example) We made these by trial and error
@@ -38,7 +38,7 @@ IDENTITIES = RemoteTable.new(:url => "file://#{File.expand_path("../../examples/
 
 FINAL_OPTIONS = {
   :haystack_reader => HAYSTACK_READER,
-  :strict_blocking => STRICT_BLOCKING,
+  :must_match_blocking => MUST_MATCH_BLOCKING,
   :tighteners => TIGHTENERS,
   :identities => IDENTITIES,
   :blockings => BLOCKINGS

data/examples/bts_aircraft/test_bts_aircraft.rb

@@ -20,7 +20,7 @@ HAYSTACK_READER = lambda { |record| "#{record['Manufacturer']} #{record['Long Na
 
 # Whether to even bother trying to find a match for something without an explicit block
 # (Example) False, which is the default, which means we have more work to do
-STRICT_BLOCKING = false
+MUST_MATCH_BLOCKING = false
 
 # Blockings
 # (Example) We made these by trial and error
@@ -64,7 +64,7 @@ NEGATIVES = RemoteTable.new :url => "file://#{File.expand_path("../negatives.csv
 
 FINAL_OPTIONS = {
   :haystack_reader => HAYSTACK_READER,
-  :strict_blocking => STRICT_BLOCKING,
+  :must_match_blocking => MUST_MATCH_BLOCKING,
   :tighteners => TIGHTENERS,
   :identities => IDENTITIES,
   :blockings => BLOCKINGS

data/lib/loose_tight_dictionary.rb

@@ -7,10 +7,10 @@ require 'active_support/version'
 }.each do |active_support_3_requirement|
   require active_support_3_requirement
 end if ::ActiveSupport::VERSION::MAJOR == 3
+require 'to_regexp'
 
 # See the README for more information.
 class LooseTightDictionary
-  autoload :ExtractRegexp, 'loose_tight_dictionary/extract_regexp'
   autoload :Tightener, 'loose_tight_dictionary/tightener'
   autoload :Blocking, 'loose_tight_dictionary/blocking'
   autoload :Identity, 'loose_tight_dictionary/identity'
@@ -48,7 +48,6 @@ class LooseTightDictionary
     find needle, options
   end
   
-  # todo fix record.record confusion (should be wrapper.record or smth)
   def find(needle, options = {})
     raise Freed if freed?
     free_last_result
@@ -69,7 +68,7 @@ class LooseTightDictionary
       last_result.needle = needle
     end
     
-    if strict_blocking and blockings.none? { |blocking| blocking.encompass? needle }
+    if must_match_blocking and blockings.any? and blockings.none? { |blocking| blocking.match? needle }
       if find_all
         return []
       else
@@ -77,25 +76,31 @@ class LooseTightDictionary
       end
     end
 
-    encompassed, unencompassed = if strict_blocking and blockings.any?
-      haystack.partition do |record|
+    encompassed, unencompassed = if blockings.any?
+      haystack.partition do |straw|
         blockings.any? do |blocking|
-          blocking.encompass?(needle, record) == true
+          blocking.encompass?(needle, straw) == true
         end
       end
     else
       [ haystack.dup, [] ]
     end
     
+    # special case: the needle didn't fit anywhere, but must_match_blocking is false, so we'll try it against everything
+    if encompassed.none?
+      encompassed = unencompassed
+      unencompassed = []
+    end
+    
     if gather_last_result
       last_result.encompassed = encompassed
       last_result.unencompassed = unencompassed
     end
     
     possibly_identical, certainly_different = if identities.any?
-      encompassed.partition do |record|
+      encompassed.partition do |straw|
         identities.all? do |identity|
-          answer = identity.identical? needle, record
+          answer = identity.identical? needle, straw
           answer.nil? or answer == true
         end
       end
@@ -109,24 +114,24 @@ class LooseTightDictionary
     end
     
     if find_all
-      return possibly_identical.map { |record| record.record }
+      return possibly_identical.map { |straw| straw.record }
     end
     
-    similarities = possibly_identical.map do |record|
-      needle.similarity record
+    similarities = possibly_identical.map do |straw|
+      needle.similarity straw
     end.sort
     
     best_similarity = similarities[-1]
-    record = best_similarity.wrapper2
+    straw = best_similarity.wrapper2
     score = best_similarity.best_score.to_f
     
     if gather_last_result
       last_result.similarities = similarities
-      last_result.record = record.record
+      last_result.record = straw.record
       last_result.score = score
     end
     
-    record.record
+    straw.record
   end
   
   # Explain is like mysql's EXPLAIN command. You give it a needle and it tells you about how it was located (successfully or not) in the haystack.
@@ -188,8 +193,8 @@ class LooseTightDictionary
     options[:haystack_reader]
   end
         
-  def strict_blocking
-    options[:strict_blocking] || false
+  def must_match_blocking
+    options[:must_match_blocking] || false
   end
 
   def tighteners

data/lib/loose_tight_dictionary/blocking.rb

@@ -7,22 +7,22 @@ class LooseTightDictionary
   # A blocking (as in a grouping) comes into effect when a str matches.
   # Then the needle must also match the blocking's regexp.
   class Blocking
-    include ExtractRegexp
-    
     attr_reader :regexp
     
     def initialize(regexp_or_str)
-      @regexp = extract_regexp regexp_or_str
+      @regexp = regexp_or_str.is_a?(::Regexp) ? regexp_or_str : regexp_or_str.to_regexp
+    end
+
+    def match?(str)
+      !!(regexp.match(str))
     end
 
     # If a blocking "encompasses" two strings, that means they both fit into it.
     #
     # Returns false if they certainly don't fit this blocking.
     # Returns nil if the blocking doesn't apply, i.e. str2 doesn't fit the blocking.
-    def encompass?(str1, str2 = nil)
-      if str2.nil?
-        !!(regexp.match(str1))
-      elsif str2_match_data = regexp.match(str2)
+    def encompass?(str1, str2)
+      if str2_match_data = regexp.match(str2)
         if str1_match_data = regexp.match(str1)
           str2_match_data.captures == str1_match_data.captures
         else

data/lib/loose_tight_dictionary/identity.rb

@@ -2,12 +2,10 @@ class LooseTightDictionary
   # Identities take effect when needle and haystack both match a regexp
   # Then the captured part of the regexp has to match exactly
   class Identity
-    include ExtractRegexp
-    
     attr_reader :regexp
     
     def initialize(regexp_or_str)
-      @regexp = extract_regexp regexp_or_str
+      @regexp = regexp_or_str.is_a?(::Regexp) ? regexp_or_str : regexp_or_str.to_regexp
     end
     
     # Two strings are "identical" if they both match this identity and the captures are equal.

data/lib/loose_tight_dictionary/tightener.rb

@@ -1,12 +1,10 @@
 class LooseTightDictionary
   # A tightener just strips a string down to its core
   class Tightener
-    include ExtractRegexp
-    
     attr_reader :regexp
     
     def initialize(regexp_or_str)
-      @regexp = extract_regexp regexp_or_str
+      @regexp = regexp_or_str.is_a?(::Regexp) ? regexp_or_str : regexp_or_str.to_regexp
     end
     
     # A tightener applies when its regexp matches and captures a new (shorter) string

data/lib/loose_tight_dictionary/version.rb

@@ -1,3 +1,3 @@
 class LooseTightDictionary
-  VERSION = '0.1.1'
+  VERSION = '0.2.0'
 end

data/loose_tight_dictionary.gemspec

@@ -23,4 +23,5 @@ Gem::Specification.new do |s|
   s.add_development_dependency "remote_table"
   s.add_dependency 'activesupport', '>=2.3.4'
   s.add_dependency 'amatch'
+  s.add_dependency 'to_regexp'
 end

data/test/test_blocking.rb

@@ -1,9 +1,9 @@
 require 'helper'
 
 class TestBlocking < Test::Unit::TestCase
-  def test_001_encompass_one
+  def test_001_match_one
     b = LooseTightDictionary::Blocking.new %r{apple}
-    assert_equal true, b.encompass?('2 apples')
+    assert_equal true, b.match?('2 apples')
   end
   
   def test_002_encompass_both

data/test/test_identity.rb

@@ -15,4 +15,19 @@ class TestIdentity < Test::Unit::TestCase
     i = LooseTightDictionary::Identity.new %r{(A)[ ]*(\d)}
     assert_equal nil, i.identical?('B1', 'A     2foobar')    
   end
+
+  def test_004_regexp
+    i = LooseTightDictionary::Identity.new %r{\A\\?/(.*)etc/mysql\$$}
+    assert_equal %r{\A\\?/(.*)etc/mysql\$$}, i.regexp
+  end
+  
+  def test_005_regexp_from_string
+    i = LooseTightDictionary::Identity.new '%r{\A\\\?/(.*)etc/mysql\$$}'
+    assert_equal %r{\A\\?/(.*)etc/mysql\$$}, i.regexp
+  end
+  
+  def test_006_regexp_from_string_using_slash_delim
+    i = LooseTightDictionary::Identity.new '/\A\\\?\/(.*)etc\/mysql\$$/'
+    assert_equal %r{\A\\?/(.*)etc/mysql\$$}, i.regexp
+  end
 end

data/test/test_loose_tight_dictionary.rb

@@ -51,24 +51,20 @@ class TestLooseTightDictionary < Test::Unit::TestCase
     assert_equal 'foo', d.find('baz')
   end
   
-  def test_009_loose_blocking
-    # sanity check
+  def test_009_must_match_blocking
     d = LooseTightDictionary.new [ 'X' ]
     assert_equal 'X', d.find('X')
     assert_equal 'X', d.find('A')
-    # end sanity check
     
     d = LooseTightDictionary.new [ 'X' ], :blockings => [ /X/, /Y/ ]
     assert_equal 'X', d.find('X')
     assert_equal 'X', d.find('A')
-  end
-  
-  def test_010_strict_blocking
-    d = LooseTightDictionary.new [ 'X' ], :blockings => [ /X/, /Y/ ], :strict_blocking => true
+    
+    d = LooseTightDictionary.new [ 'X' ], :blockings => [ /X/, /Y/ ], :must_match_blocking => true
     assert_equal 'X', d.find('X')
     assert_equal nil, d.find('A')
   end
-  
+    
   def test_011_free
     d = LooseTightDictionary.new %w{ NISSAN HONDA }
     d.free
@@ -78,7 +74,7 @@ class TestLooseTightDictionary < Test::Unit::TestCase
   end
   
   def test_012_find_all
-    d = LooseTightDictionary.new [ 'X', 'X22', 'Y', 'Y4' ], :blockings => [ /X/, /Y/ ], :strict_blocking => true
+    d = LooseTightDictionary.new [ 'X', 'X22', 'Y', 'Y4' ], :blockings => [ /X/, /Y/ ], :must_match_blocking => true
     assert_equal ['X', 'X22' ], d.find_all('X')
     assert_equal [], d.find_all('A')
   end

metadata

@@ -1,8 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: loose_tight_dictionary
 version: !ruby/object:Gem::Version 
+  hash: 23
   prerelease: 
-  version: 0.1.1
+  segments: 
+  - 0
+  - 2
+  - 0
+  version: 0.2.0
 platform: ruby
 authors: 
 - Seamus Abshere
@@ -10,8 +15,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2011-04-18 00:00:00 -05:00
-default_executable: 
+date: 2011-04-27 00:00:00 Z
 dependencies: 
 - !ruby/object:Gem::Dependency 
   name: shoulda
@@ -21,6 +25,9 @@ dependencies:
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
         version: "0"
   type: :development
   version_requirements: *id001
@@ -32,6 +39,9 @@ dependencies:
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
         version: "0"
   type: :development
   version_requirements: *id002
@@ -43,6 +53,11 @@ dependencies:
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
+        hash: 11
+        segments: 
+        - 2
+        - 3
+        - 4
         version: 2.3.4
   type: :runtime
   version_requirements: *id003
@@ -54,9 +69,26 @@ dependencies:
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
         version: "0"
   type: :runtime
   version_requirements: *id004
+- !ruby/object:Gem::Dependency 
+  name: to_regexp
+  prerelease: false
+  requirement: &id005 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  type: :runtime
+  version_requirements: *id005
 description: Create dictionaries that link rows between two tables using loose matching (string similarity) by default and tight matching (regexp) by request.
 email: 
 - seamus@abshere.net
@@ -93,7 +125,6 @@ files:
 - examples/icao-bts.xls
 - lib/loose_tight_dictionary.rb
 - lib/loose_tight_dictionary/blocking.rb
-- lib/loose_tight_dictionary/extract_regexp.rb
 - lib/loose_tight_dictionary/identity.rb
 - lib/loose_tight_dictionary/result.rb
 - lib/loose_tight_dictionary/score.rb
@@ -104,12 +135,10 @@ files:
 - loose_tight_dictionary.gemspec
 - test/helper.rb
 - test/test_blocking.rb
-- test/test_extract_regexp.rb
 - test/test_identity.rb
 - test/test_loose_tight_dictionary.rb
 - test/test_loose_tight_dictionary_convoluted.rb.disabled
 - test/test_tightening.rb
-has_rdoc: true
 homepage: https://github.com/seamusabshere/loose_tight_dictionary
 licenses: []
 
@@ -123,24 +152,29 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements: 
   - - ">="
     - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
       version: "0"
 required_rubygems_version: !ruby/object:Gem::Requirement 
   none: false
   requirements: 
   - - ">="
     - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
       version: "0"
 requirements: []
 
 rubyforge_project: loose_tight_dictionary
-rubygems_version: 1.6.2
+rubygems_version: 1.7.2
 signing_key: 
 specification_version: 3
 summary: Allows iterative development of dictionaries for big data sets.
 test_files: 
 - test/helper.rb
 - test/test_blocking.rb
-- test/test_extract_regexp.rb
 - test/test_identity.rb
 - test/test_loose_tight_dictionary.rb
 - test/test_loose_tight_dictionary_convoluted.rb.disabled

data/lib/loose_tight_dictionary/extract_regexp.rb

@@ -1,30 +0,0 @@
-class LooseTightDictionary
-  module ExtractRegexp #:nodoc: all
-    def extract_regexp(regexp_or_str)
-      case regexp_or_str
-      when ::Regexp
-        regexp_or_str
-      when ::String
-        regexp_from_string regexp_or_str
-      else
-        raise ::ArgumentError, "Expected regexp or string"
-      end
-    end
-    
-    REGEXP_DELIMITERS = {
-      '%r{' => '}',
-      '/' => '/'
-    }
-    def regexp_from_string(str)
-      delim_start, delim_end = REGEXP_DELIMITERS.detect { |k, v| str.start_with? k }.map { |delim| ::Regexp.escape delim }
-      %r{\A#{delim_start}(.*)#{delim_end}([^#{delim_end}]*)\z} =~ str.strip
-      content = $1
-      options = $2
-      content.gsub! '\\/', '/'
-      ignore_case = options.include?('i') ? ::Regexp::IGNORECASE : nil
-      multiline = options.include?('m') ? ::Regexp::MULTILINE : nil
-      extended = options.include?('x') ? ::Regexp::EXTENDED : nil
-      ::Regexp.new content, (ignore_case||multiline||extended)
-    end
-  end
-end

data/test/test_extract_regexp.rb

@@ -1,18 +0,0 @@
-require 'helper'
-
-class TestExtractRegexp < Test::Unit::TestCase
-  def test_001_regexp
-    i = LooseTightDictionary::Identity.new %r{\A\\?/(.*)etc/mysql\$$}
-    assert_equal %r{\A\\?/(.*)etc/mysql\$$}, i.regexp
-  end
-  
-  def test_002_regexp_from_string
-    i = LooseTightDictionary::Identity.new '%r{\A\\\?/(.*)etc/mysql\$$}'
-    assert_equal %r{\A\\?/(.*)etc/mysql\$$}, i.regexp
-  end
-  
-  def test_003_regexp_from_string_using_slash_delim
-    i = LooseTightDictionary::Identity.new '/\A\\\?\/(.*)etc\/mysql\$$/'
-    assert_equal %r{\A\\?/(.*)etc/mysql\$$}, i.regexp
-  end
-end