loose_tight_dictionary 0.2.3 → 1.0.0

data/LICENSE

@@ -1,4 +1,4 @@
-Copyright (c) 2009 Seamus Abshere
+Copyright 2011 Brighter Planet, Inc.
 
 Permission is hereby granted, free of charge, to any person obtaining
 a copy of this software and associated documentation files (the

data/README.rdoc

@@ -4,102 +4,59 @@ Match things based on string similarity (using the Pair Distance algorithm) and
 
 == Quickstart
 
-    >> d = LooseTightDictionary.new %w{seamus andy ben}
-    => [...]
-    >> puts d.find 'Shamus Heaney'
-    => 'seamus'
-
-Try running the included example file:
-
-    $ ruby examples/first_name_matching.rb 
-    ######################################################################################################################################################
-    # Match "Mr. Seamus" => "seamus"
-    ######################################################################################################################################################
-
-    Needle
-    (needle_reader proc not defined, so downcasing everything)
-    ------------------------------------------------------------------------------------------------------------------------------------------------------
-    "mr. seamus"
-
-    Haystack
-    (haystack_reader proc not defined, so downcasing everything)
-    ------------------------------------------------------------------------------------------------------------------------------------------------------
-    "seamus"
-    "andy"
-    "ben"
-
-    Tighteners
-    ------------------------------------------------------------------------------------------------------------------------------------------------------
-    (none)
-
-    Comparisons
-    Score                                             t_haystack [=> tightened/prefixed]                t_needle [=> tightened/prefixed]                  
-    ------------------------------------------------------------------------------------------------------------------------------------------------------
-    0.8333333333333334                                "seamus"                                          "mr. seamus"
-    0.0                                               "andy"                                            "mr. seamus"
-    0.0                                               "ben"                                             "mr. seamus"
-
-    Match
-    ------------------------------------------------------------------------------------------------------------------------------------------------------
-    "seamus"
-
-    # [... there's more output ...]
-
-== The Boeing example
-
-From the tests:
-
-    ######################################################################################################################################################
-    # Match "BOEING 737100" => "BOEING BOEING 737-100/200"
-    ######################################################################################################################################################
-
-    Needle
-    (needle_reader proc not defined, so downcasing everything)
-    ------------------------------------------------------------------------------------------------------------------------------------------------------
-    "boeing 737100"
-
-    Haystack
-    (haystack_reader proc not defined, so downcasing everything)
-    ------------------------------------------------------------------------------------------------------------------------------------------------------
-    "boeing boeing 737-100/200"
-    "boeing boeing 737-900"
-
-    Tighteners
-    ------------------------------------------------------------------------------------------------------------------------------------------------------
-    /(7\d)(7|0)-?(\d{1,3})/i
-
-    Comparisons
-    Score                                             t_haystack [=> tightened/prefixed]                t_needle [=> tightened/prefixed]                  
-    ------------------------------------------------------------------------------------------------------------------------------------------------------
-    1.0                                               "boeing boeing 737-100/200" => "737100"           "boeing 737100" => "737100"
-    0.6666666666666666                                "boeing boeing 737-100/200" => "737100"           "boeing 737100"
-    0.6153846153846154                                "boeing boeing 737-900"                           "boeing 737100"
-    0.6                                               "boeing boeing 737-900" => "737900"               "boeing 737100" => "737100"
-    0.6                                               "boeing boeing 737-100/200"                       "boeing 737100"
-    0.4                                               "boeing boeing 737-900" => "737900"               "boeing 737100"
-    0.32                                              "boeing boeing 737-100/200"                       "boeing 737100" => "737100"
-    0.2857142857142857                                "boeing boeing 737-900"                           "boeing 737100" => "737100"
-
-    Match
-    ------------------------------------------------------------------------------------------------------------------------------------------------------
-    "BOEING BOEING 737-100/200"
-
-== Improving dictionaries
-
-Similarity matching will only get you so far.
-
-    TODO: regex usage
-
-== Note on Patches/Pull Requests
- 
-* Fork the project.
-* Make your feature addition or bug fix.
-* Add tests for it. This is important so I don't break it in a
-  future version unintentionally.
-* Commit, do not mess with rakefile, version, or history.
-  (if you want to have your own version, that is fine but bump version in a commit by itself I can ignore when I pull)
-* Send me a pull request. Bonus points for topic branches.
+    >> require 'loose_tight_dictionary'
+    => true 
+    >> LooseTightDictionary.new(%w{seamus andy ben}).find('Shamus')
+    => "seamus"
+
+== String similarity matching
+
+Exclusively uses {Dice's Coefficient}[http://en.wikipedia.org/wiki/Dice's_coefficient] algorithm (aka Pair Distance).
+
+== Production use
+
+Over 2 years in {Brighter Planet's environmental impact API}[http://impact.brighterplanet.com] and {reference data service}[http://data.brighterplanet.com].
+
+== Speed
+
+If you add the amatch[http://flori.github.com/amatch/] gem to your Gemfile, it will use that, which is much faster (but {segfaults have been seen in the wild}[https://github.com/flori/amatch/issues/3]).
+
+Otherwise, a {pure ruby version}[http://stackoverflow.com/questions/653157/a-better-similarity-ranking-algorithm-for-variable-length-strings] is used.
+
+== Regular expressions
+
+You can improve the default matchings with regular expressions.
+
+* Emphasize important words using <b>blockings</b> and <b>tighteners</b>
+* Filter out stop words with <b>tighteners</b>
+* Prevent impossible matches with <b>blockings</b> and <b>identities</b>
+
+=== Blockings
+
+Setting a blocking of <tt>/Airbus/</tt> ensures that strings containing "Airbus" will only be scored against to other strings containing "Airbus". A better blocking in this case would probably be <tt>/airbus/i</tt>.
+
+=== Tighteners
+
+Adding a tightener like <tt>/(boeing).*(7\d\d)/i</tt> will cause "BOEING COMPANY 747" and "boeing747" to be scored as if they were "BOEING 747" and "boeing 747", respectively. See also "Case sensitivity" below.
+
+=== Identities
+
+Adding an identity like <tt>/(F)\-?(\d50)/</tt> ensures that "Ford F-150" and "Ford F-250" never match.
+
+== Case sensitivity
+
+Scoring is case-insensitive. Everything is downcased before scoring. This is a change from previous versions.
+
+== Examples
+
+Try running the included example files (<tt>examples/first_name_matching.rb</tt>) and check out the tests.
+
+== Authors
+
+* Seamus Abshere <seamus@abshere.net>
+* Ian Hough <ijhough@gmail.com>
+* Andy Rossmeissl <andy@rossmeissl.net>
 
 == Copyright
 
-Copyright (c) 2011 Seamus Abshere. See LICENSE for details.
+Copyright 2011 Brighter Planet, Inc.

data/THANKS-WILLIAM-JAMES.rb

@@ -0,0 +1,37 @@
+#!/usr/bin/env ruby
+
+# Thanks William James!
+# http://www.ruby-forum.com/topic/95519#200484
+def cart_prod(*args)
+  args.inject([[]]){|old,lst|
+    new = []
+    lst.each{|e| new += old.map{|c| c.dup << e }}
+    new
+  }
+end
+
+require 'benchmark'
+
+a = [1,2,3]
+b = [4,5]
+Benchmark.bmbm do |x|
+  x.report("native") do
+    500_000.times { a.product(b) }
+  end
+  x.report("william-james") do |x|
+    500_000.times { cart_prod(a, b) }
+  end
+end
+
+# results:
+# $ ruby foo.rb 
+# Rehearsal -------------------------------------------------
+# native          0.720000   0.000000   0.720000 (  0.729319)
+# william-james   3.620000   0.010000   3.630000 (  3.629198)
+# ---------------------------------------- total: 4.350000sec
+# 
+#                     user     system      total        real
+# native          0.710000   0.000000   0.710000 (  0.708620)
+# william-james   3.800000   0.000000   3.800000 (  3.792538)
+
+# thanks for all the fish!

data/lib/loose_tight_dictionary/score.rb

@@ -1,4 +1,8 @@
-require 'amatch'
+begin
+  require 'amatch'
+rescue ::LoadError
+  # using native ruby similarity scoring
+end
 
 class LooseTightDictionary
   class Score
@@ -10,7 +14,7 @@ class LooseTightDictionary
     end
     
     def to_f
-      @to_f ||= str1.pair_distance_similar str2
+      @to_f ||= dices_coefficient(str1, str2)
     end
     
     def inspect
@@ -24,5 +28,44 @@ class LooseTightDictionary
     def ==(other)
       to_f == other.to_f
     end
+    
+    private
+    
+    # http://stackoverflow.com/questions/653157/a-better-similarity-ranking-algorithm-for-variable-length-strings
+    if defined?(::Amatch)
+      def dices_coefficient(str1, str2)
+        str1 = str1.downcase 
+        str2 = str2.downcase
+        str1.pair_distance_similar str2
+      end
+    else
+      SPACE = ' '
+      def dices_coefficient(str1, str2)
+        str1 = str1.downcase 
+        str2 = str2.downcase
+        pairs1 = (0..str1.length-2).map do |i|
+          str1[i,2]
+        end.reject do |pair|
+          pair.include? SPACE
+        end
+        pairs2 = (0..str2.length-2).map do |i|
+          str2[i,2]
+        end.reject do |pair|
+          pair.include? SPACE
+        end
+        union = pairs1.size + pairs2.size 
+        intersection = 0 
+        pairs1.each do |p1| 
+          0.upto(pairs2.size-1) do |i| 
+            if p1 == pairs2[i] 
+              intersection += 1 
+              pairs2.slice!(i) 
+              break 
+            end 
+          end 
+        end 
+        (2.0 * intersection) / union
+      end
+    end
   end
 end

data/lib/loose_tight_dictionary/similarity.rb

@@ -34,7 +34,7 @@ class LooseTightDictionary
     end
     
     def best_variants
-      @best_variants ||= cart_prod(wrapper1.variants, wrapper2.variants).sort do |tuple1, tuple2|
+      @best_variants ||= wrapper1.variants.product(wrapper2.variants).sort do |tuple1, tuple2|
         wrapper1_variant1, wrapper2_variant1 = tuple1
         wrapper1_variant2, wrapper2_variant2 = tuple2
 
@@ -48,15 +48,5 @@ class LooseTightDictionary
     def inspect
       %{#<Similarity "#{wrapper2.to_str}"=>"#{best_wrapper2_variant}" versus "#{wrapper1.to_str}"=>"#{best_wrapper1_variant}" weight=#{"%0.5f" % weight} best_score=#{"%0.5f" % best_score.to_f}>}
     end
-    
-    # Thanks William James!
-    # http://www.ruby-forum.com/topic/95519#200484
-    def cart_prod(*args)
-      args.inject([[]]){|old,lst|
-        new = []
-        lst.each{|e| new += old.map{|c| c.dup << e }}
-        new
-      }
-    end
   end
 end

data/lib/loose_tight_dictionary/version.rb

@@ -1,3 +1,3 @@
 class LooseTightDictionary
-  VERSION = '0.2.3'
+  VERSION = '1.0.0'
 end

data/loose_tight_dictionary.gemspec

@@ -25,7 +25,8 @@ Gem::Specification.new do |s|
   s.add_development_dependency 'mysql'
   s.add_development_dependency 'cohort_scope'
   s.add_development_dependency 'weighted_average'
-  s.add_dependency 'activesupport', '>=3'
-  s.add_dependency 'amatch'
-  s.add_dependency 'to_regexp', '>=0.0.3'
+  s.add_development_dependency 'rake'
+  # s.add_development_dependency 'amatch'
+  s.add_runtime_dependency 'activesupport', '>=3'
+  s.add_runtime_dependency 'to_regexp', '>=0.0.3'
 end

data/test/test_cache.rb

@@ -112,7 +112,7 @@ class TestCache < Test::Unit::TestCase
   
   def test_004_weighted_average
     aircraft = Aircraft.find('B742')
-    assert_equal 5.4545, aircraft.flight_segments.weighted_average(:seats, :weighted_by => :passengers)
+    assert_equal 5.45455, aircraft.flight_segments.weighted_average(:seats, :weighted_by => :passengers)
   end
   
   def test_005_right_way_to_do_cohorts

metadata

@@ -1,133 +1,130 @@
---- !ruby/object:Gem::Specification 
+--- !ruby/object:Gem::Specification
 name: loose_tight_dictionary
-version: !ruby/object:Gem::Version 
+version: !ruby/object:Gem::Version
+  version: 1.0.0
   prerelease: 
-  version: 0.2.3
 platform: ruby
-authors: 
+authors:
 - Seamus Abshere
 autorequire: 
 bindir: bin
 cert_chain: []
-
-date: 2011-05-17 00:00:00 -05:00
-default_executable: 
-dependencies: 
-- !ruby/object:Gem::Dependency 
+date: 2011-12-03 00:00:00.000000000Z
+dependencies:
+- !ruby/object:Gem::Dependency
   name: shoulda
-  prerelease: false
-  requirement: &id001 !ruby/object:Gem::Requirement 
+  requirement: &2160898300 !ruby/object:Gem::Requirement
     none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        version: "0"
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
   type: :development
-  version_requirements: *id001
-- !ruby/object:Gem::Dependency 
-  name: remote_table
   prerelease: false
-  requirement: &id002 !ruby/object:Gem::Requirement 
+  version_requirements: *2160898300
+- !ruby/object:Gem::Dependency
+  name: remote_table
+  requirement: &2160965820 !ruby/object:Gem::Requirement
     none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        version: "0"
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
   type: :development
-  version_requirements: *id002
-- !ruby/object:Gem::Dependency 
-  name: activerecord
   prerelease: false
-  requirement: &id003 !ruby/object:Gem::Requirement 
+  version_requirements: *2160965820
+- !ruby/object:Gem::Dependency
+  name: activerecord
+  requirement: &2161059600 !ruby/object:Gem::Requirement
     none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        version: "3"
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '3'
   type: :development
-  version_requirements: *id003
-- !ruby/object:Gem::Dependency 
-  name: mysql
   prerelease: false
-  requirement: &id004 !ruby/object:Gem::Requirement 
+  version_requirements: *2161059600
+- !ruby/object:Gem::Dependency
+  name: mysql
+  requirement: &2161138940 !ruby/object:Gem::Requirement
     none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        version: "0"
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
   type: :development
-  version_requirements: *id004
-- !ruby/object:Gem::Dependency 
-  name: cohort_scope
   prerelease: false
-  requirement: &id005 !ruby/object:Gem::Requirement 
+  version_requirements: *2161138940
+- !ruby/object:Gem::Dependency
+  name: cohort_scope
+  requirement: &2161208000 !ruby/object:Gem::Requirement
     none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        version: "0"
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
   type: :development
-  version_requirements: *id005
-- !ruby/object:Gem::Dependency 
-  name: weighted_average
   prerelease: false
-  requirement: &id006 !ruby/object:Gem::Requirement 
+  version_requirements: *2161208000
+- !ruby/object:Gem::Dependency
+  name: weighted_average
+  requirement: &2161382600 !ruby/object:Gem::Requirement
     none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        version: "0"
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
   type: :development
-  version_requirements: *id006
-- !ruby/object:Gem::Dependency 
-  name: activesupport
   prerelease: false
-  requirement: &id007 !ruby/object:Gem::Requirement 
+  version_requirements: *2161382600
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: &2161619140 !ruby/object:Gem::Requirement
     none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        version: "3"
-  type: :runtime
-  version_requirements: *id007
-- !ruby/object:Gem::Dependency 
-  name: amatch
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
   prerelease: false
-  requirement: &id008 !ruby/object:Gem::Requirement 
+  version_requirements: *2161619140
+- !ruby/object:Gem::Dependency
+  name: activesupport
+  requirement: &2161769980 !ruby/object:Gem::Requirement
     none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        version: "0"
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '3'
   type: :runtime
-  version_requirements: *id008
-- !ruby/object:Gem::Dependency 
-  name: to_regexp
   prerelease: false
-  requirement: &id009 !ruby/object:Gem::Requirement 
+  version_requirements: *2161769980
+- !ruby/object:Gem::Dependency
+  name: to_regexp
+  requirement: &2161790420 !ruby/object:Gem::Requirement
     none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
         version: 0.0.3
   type: :runtime
-  version_requirements: *id009
-description: Create dictionaries that link rows between two tables using loose matching (string similarity) by default and tight matching (regexp) by request.
-email: 
+  prerelease: false
+  version_requirements: *2161790420
+description: Create dictionaries that link rows between two tables using loose matching
+  (string similarity) by default and tight matching (regexp) by request.
+email:
 - seamus@abshere.net
 executables: []
-
 extensions: []
-
 extra_rdoc_files: []
-
-files: 
+files:
 - .document
 - .gitignore
 - Gemfile
 - LICENSE
 - README.rdoc
 - Rakefile
+- THANKS-WILLIAM-JAMES.rb
 - benchmark/before-with-free.txt
 - benchmark/before-without-last-result.txt
 - benchmark/before.txt
@@ -164,35 +161,31 @@ files:
 - test/test_loose_tight_dictionary.rb
 - test/test_loose_tight_dictionary_convoluted.rb.disabled
 - test/test_tightening.rb
-has_rdoc: true
 homepage: https://github.com/seamusabshere/loose_tight_dictionary
 licenses: []
-
 post_install_message: 
 rdoc_options: []
-
-require_paths: 
+require_paths:
 - lib
-required_ruby_version: !ruby/object:Gem::Requirement 
+required_ruby_version: !ruby/object:Gem::Requirement
   none: false
-  requirements: 
-  - - ">="
-    - !ruby/object:Gem::Version 
-      version: "0"
-required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
-  requirements: 
-  - - ">="
-    - !ruby/object:Gem::Version 
-      version: "0"
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
 requirements: []
-
 rubyforge_project: loose_tight_dictionary
-rubygems_version: 1.6.2
+rubygems_version: 1.8.10
 signing_key: 
 specification_version: 3
 summary: Allows iterative development of dictionaries for big data sets.
-test_files: 
+test_files:
 - test/helper.rb
 - test/test_blocking.rb
 - test/test_cache.rb