fuzzy_match 1.3.2 → 1.3.3

data/CHANGELOG

@@ -0,0 +1,21 @@
+== 1.3.3 / 2012-04-13
+
+* Enhancements
+
+  * Now you must require 'fuzzy_match/cached_result' if you want to use it.
+  * Use active_record_inline_schema to create the FuzzyMatch::CachedResult table
+  * Test against CohortAnalysis, the replacement for CohortScope
+  * Fix some other random deprecations (like set_primary_key)
+
+== 1.3.2 / 2012-02-24
+
+* Enhancements
+
+  * Start keeping a changelog!
+  * renamed blockings to groupings
+  * cleaned up tests
+
+* Bug fixes
+
+  * better handling for one-letter similiarities like 'X foo' vs 'X bar' which couldn't be detected by pair distance
+  * take deprecated option :tighteners as :normalizers

data/Gemfile

@@ -2,12 +2,16 @@ source :rubygems
 
 gemspec
 
+# bin dependencies
+gem 'remote_table'
+gem 'thor'
+
 # development dependencies
 gem 'minitest-reporters'
 gem "minitest"
 gem 'activerecord', '>=3'
 gem 'mysql2'
-gem 'cohort_scope'
+gem 'cohort_analysis'
 gem 'weighted_average'
 gem 'rake'
 gem 'yard'

data/README.markdown

@@ -8,9 +8,7 @@ Replaces [`loose_tight_dictionary`](https://github.com/seamusabshere/loose_tight
 
     >> require 'fuzzy_match'
     => true
-    >> matcher = FuzzyMatch.new(['seamus', 'andy', 'ben'])
-    => #<FuzzyMatch: [...]>
-    >> matcher.find('Shamus')
+    >> FuzzyMatch.new(['seamus', 'andy', 'ben']).find('Shamus)
     => "seamus"
 
 See also the blog post [Fuzzy match in Ruby](http://numbers.brighterplanet.com/2012/01/18/fuzzy-match-in-ruby/).
@@ -118,7 +116,7 @@ In edge cases where Dice's finds that two strings are equally similar to a third
 
 ## Production use
 
-Over 2 years in [Brighter Planet's environmental impact API](http://impact.brighterplanet.com) and [reference data service](http://data.brighterplanet.com).
+Over 2 years in [Brighter Planet's impact estimate API](http://impact.brighterplanet.com) and [reference data service](http://data.brighterplanet.com).
 
 We often combine `fuzzy_match` with [`remote_table`](https://github.com/seamusabshere/remote_table) and [`errata`](https://github.com/seamusabshere/errata):
 
@@ -126,6 +124,10 @@ We often combine `fuzzy_match` with [`remote_table`](https://github.com/seamusab
 - correct serious or repeated errors with `errata`
 - `fuzzy_match` the rest
 
+## Cached results
+
+TODO write documentation. For now, please see how [we manually cache matches between aircraft and flight segments](https://github.com/brighterplanet/earth/blob/master/lib/earth/air/aircraft.rb).
+
 ## Glossary
 
 The admittedly imperfect metaphor is "look for a needle in a haystack"

data/bin/fuzzy_match_checker

@@ -0,0 +1,66 @@
+#!/usr/bin/env ruby
+
+if File.exist?(File.join(Dir.pwd, 'fuzzy_match.gemspec'))
+  require 'bundler/setup'
+end
+
+require 'fuzzy_match'
+require 'active_support/core_ext'
+require 'remote_table'
+require 'thor'
+
+class FuzzyMatch
+  class Checker < ::Thor
+    # for example: https://docs.google.com/spreadsheet/pub?key=0AkCJNpm9Ks6JdHZURUI2S2xOa3ZFVzlZb205VVhpQnc&single=true&gid=0&output=csv
+    desc :check, "Check a spreadsheet containing columns with these headers: haystack, needles, correct_matches, groupings, stop_words, identities, normalizers, find_options (listing an option like must_match_grouping makes it true)"
+    method_option :show_success, :default => false, :type => :boolean, :desc => "Whether to print successful matches as you go"
+    method_option :downcase, :default => false, :type => :boolean, :desc => "Whether to downcase everything (except regexes, where you have to do /foo/i)"
+    def check(url)
+      puts "Checking matches using fuzzy_match version #{FuzzyMatch::VERSION}..."
+
+      t = RemoteTable.new(url, :headers => :first_row)
+      if (violators = %w{needle grouping correct_match stop_word identity normalizer find_option} & t.rows.first.keys).any?
+        raise ArgumentError, "Make sure you pluralize your right row headers (violators: #{violators.map(&:inspect).join(', ')}"
+      end
+      haystack = t.rows.map { |row| row['haystack'] }.select(&:present?)
+      haystack.map!(&:downcase) if options.downcase
+      find_options = t.rows.map { |row| row['find_options'] }
+      fm = FuzzyMatch.new(
+        haystack,
+        :groupings => t.rows.map { |row| row['groupings'] }.select(&:present?),
+        :identities => t.rows.map { |row| row['identities'] }.select(&:present?),
+        :stop_words => t.rows.map { |row| row['stop_words'] }.select(&:present?),
+        :normalizers => t.rows.map { |row| row['normalizers'] }.select(&:present?),
+        :must_match_grouping => find_options.include?('must_match_grouping'),
+        :must_match_at_least_one_word => find_options.include?('must_match_at_least_one_word'),
+        :first_grouping_decides => find_options.include?('first_grouping_decides')
+      )
+
+      count = 0
+      t.each do |row|
+        needle = row['needles']
+        correct_match = row['correct_matches']
+        next unless needle.present?
+        if options.downcase
+          needle.to_s.downcase!
+          correct_match.to_s.downcase!
+        end
+        correct_match = nil if correct_match.blank?
+        match = fm.find needle
+        if options.show_success? or match != correct_match
+          puts "  #{needle.inspect} => #{match.inspect}"
+        end
+        unless match == correct_match
+          puts "MISMATCH: #{needle.inspect} should match #{correct_match.inspect}"
+          exit 1
+        end
+        count += 1
+      end
+
+      puts "Correctly matched #{count} needles."
+      exit 0
+    end
+  end
+end
+
+FuzzyMatch::Checker.start

data/fuzzy_match.gemspec

@@ -1,11 +1,9 @@
 # -*- encoding: utf-8 -*-
-$:.push File.expand_path("../lib", __FILE__)
-require "fuzzy_match/version"
+require File.expand_path("../lib/fuzzy_match/version", __FILE__)
 
 Gem::Specification.new do |s|
   s.name        = "fuzzy_match"
   s.version     = FuzzyMatch::VERSION
-  s.platform    = Gem::Platform::RUBY
   s.authors     = ["Seamus Abshere"]
   s.email       = ["seamus@abshere.net"]
   s.homepage    = "https://github.com/seamusabshere/fuzzy_match"
@@ -21,4 +19,5 @@ Gem::Specification.new do |s|
 
   s.add_runtime_dependency 'activesupport', '>=3'
   s.add_runtime_dependency 'to_regexp', '>=0.0.3'
+  s.add_runtime_dependency 'active_record_inline_schema', '>=0.4.0'
 end

data/lib/fuzzy_match/cached_result.rb

@@ -1,3 +1,5 @@
+require 'active_record_inline_schema'
+
 class FuzzyMatch
   class CachedResult < ::ActiveRecord::Base
     if ::ActiveRecord::VERSION::STRING >= '3.2'
@@ -5,26 +7,23 @@ class FuzzyMatch
     else
       set_table_name :fuzzy_match_cached_results
     end
-    
-    def self.create_table
-      connection.create_table :fuzzy_match_cached_results do |t|
-        t.string :a_class
-        t.string :a
-        t.string :b_class
-        t.string :b
-      end
-      connection.add_index :fuzzy_match_cached_results, [:a_class, :b_class, :a], :name => 'aba'
-      connection.add_index :fuzzy_match_cached_results, [:a_class, :b_class, :b], :name => 'abb'
-      connection.add_index :fuzzy_match_cached_results, [:a_class, :b_class, :a, :b], :name => 'abab'
-      reset_column_information
-    end
-    
-    def self.setup(from_scratch = false)
-      if from_scratch or not table_exists?
-        connection.drop_table :fuzzy_match_cached_results rescue nil
-        create_table rescue nil
+
+    class << self
+      def setup(from_scratch = false)
+        if from_scratch
+          connection.drop_table :fuzzy_match_cached_results rescue nil
+        end
+        auto_upgrade!
       end
     end
+
+    col :a_class
+    col :a
+    col :b_class
+    col :b
+    add_index [:a_class, :b_class, :a], :name => 'aba'
+    add_index [:a_class, :b_class, :b], :name => 'abb'
+    add_index [:a_class, :b_class, :a, :b], :name => 'abab'
     
     module ActiveRecordBaseExtension
       # required options:

data/lib/fuzzy_match/version.rb

@@ -1,3 +1,3 @@
 class FuzzyMatch
-  VERSION = '1.3.2'
+  VERSION = '1.3.3'
 end

data/lib/fuzzy_match.rb

@@ -15,10 +15,6 @@ require 'fuzzy_match/wrapper'
 require 'fuzzy_match/similarity'
 require 'fuzzy_match/score'
 
-if defined?(::ActiveRecord)
-  require 'fuzzy_match/cached_result'
-end
-
 # See the README for more information.
 class FuzzyMatch
   class << self

data/test/test_amatch.rb

@@ -3,14 +3,16 @@ unless RUBY_PLATFORM == 'java'
   require 'test_fuzzy_match'
   require 'amatch'
   
-  class TestAmatch < TestFuzzyMatch
-    before do
-      $testing_amatch = true
-      FuzzyMatch.engine = :amatch
-    end
-    after do
-      $testing_amatch = false
-      FuzzyMatch.engine = nil
+  describe FuzzyMatch do
+    describe %{when using the :amatch string similarity engine} do
+      before do
+        $testing_amatch = true
+        FuzzyMatch.engine = :amatch
+      end
+      after do
+        $testing_amatch = false
+        FuzzyMatch.engine = nil
+      end
     end
   end
 end

data/test/test_cache.rb

@@ -2,7 +2,7 @@ require 'helper'
 
 require 'active_support/all'
 require 'active_record'
-require 'cohort_scope'
+require 'cohort_analysis'
 require 'weighted_average'
 
 ActiveRecord::Base.establish_connection(
@@ -25,7 +25,7 @@ require 'fuzzy_match/cached_result'
 ::FuzzyMatch::CachedResult.setup(true)
 
 class Aircraft < ActiveRecord::Base
-  set_primary_key :icao_code
+  self.primary_key = 'icao_code'
   
   cache_fuzzy_match_with :flight_segments, :primary_key => :aircraft_description, :foreign_key => :aircraft_description
     
@@ -52,13 +52,10 @@ CREATE TABLE `aircraft` (
 end
 
 class FlightSegment < ActiveRecord::Base
-  set_primary_key :row_hash
+  self.primary_key = 'row_hash'
   
   cache_fuzzy_match_with :aircraft, :primary_key => :aircraft_description, :foreign_key => :aircraft_description
   
-  extend CohortScope
-  self.minimum_cohort_size = 1
-  
   def self.create_table
     connection.drop_table(:flight_segments) rescue nil
     connection.execute %{
@@ -96,11 +93,11 @@ fs.passengers = 100
 fs.seats = 5
 fs.save!
 
-FlightSegment.find_each do |fs|
+FlightSegment.all.each do |fs|
   fs.cache_aircraft!
 end
 
-class TestCache < MiniTest::Spec
+describe FuzzyMatch::CachedResult do
   it %{joins aircraft to flight segments} do
     aircraft = Aircraft.find('B742')
     aircraft.flight_segments.count.must_equal 2
@@ -118,7 +115,8 @@ class TestCache < MiniTest::Spec
   
   it %{works with cohort_scope (albeit rather clumsily)} do
     aircraft = Aircraft.find('B742')
-    FlightSegment.big_cohort(:aircraft_description => aircraft.flight_segments_foreign_keys).count.must_equal 2
+    FlightSegment.cohort({:aircraft_description => aircraft.flight_segments_foreign_keys}, :minimum_size => 2).count.must_equal 2
+    # FlightSegment.cohort(:aircraft_description => aircraft.flight_segments_foreign_keys).must_equal []
   end
   
   # def test_006_you_can_get_aircraft_from_flight_segments

data/test/test_fuzzy_match.rb

@@ -1,7 +1,7 @@
 # -*- encoding: utf-8 -*-
 require 'helper'
 
-class TestFuzzyMatch < MiniTest::Spec
+describe FuzzyMatch do
   describe '#find' do
     it %{identifies the best match based on string similarity} do
       d = FuzzyMatch.new %w{ RATZ CATZ }
@@ -12,6 +12,11 @@ class TestFuzzyMatch < MiniTest::Spec
       d.find('X').must_equal 'X'
       d.find('A').must_be_nil
     end
+
+    it %{does the right thing} do
+      d = FuzzyMatch.new [ 'Artyom Makarov', 'Karl' ], :must_match_at_least_one_word => true
+      puts d.explain('art')#.must_equal 'Artyom Makarov'
+    end
   
     it %{not return any result if the maximum score is zero} do
       FuzzyMatch.new(['a']).find('b').must_be_nil

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fuzzy_match
 version: !ruby/object:Gem::Version
-  version: 1.3.2
+  version: 1.3.3
   prerelease: 
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-02-24 00:00:00.000000000 Z
+date: 2012-04-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
-  requirement: &2151876580 !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -21,10 +21,15 @@ dependencies:
         version: '3'
   type: :runtime
   prerelease: false
-  version_requirements: *2151876580
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '3'
 - !ruby/object:Gem::Dependency
   name: to_regexp
-  requirement: &2151871760 !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -32,19 +37,41 @@ dependencies:
         version: 0.0.3
   type: :runtime
   prerelease: false
-  version_requirements: *2151871760
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 0.0.3
+- !ruby/object:Gem::Dependency
+  name: active_record_inline_schema
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 0.4.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 0.4.0
 description: Find a needle in a haystack using string similarity and (optionally)
   regexp rules. Replaces loose_tight_dictionary.
 email:
 - seamus@abshere.net
-executables: []
+executables:
+- fuzzy_match_checker
 extensions: []
 extra_rdoc_files: []
 files:
 - .document
 - .gitignore
+- CHANGELOG
 - Gemfile
-- History.txt
 - LICENSE
 - README.markdown
 - Rakefile
@@ -53,6 +80,7 @@ files:
 - benchmark/before-without-last-result.txt
 - benchmark/before.txt
 - benchmark/memory.rb
+- bin/fuzzy_match_checker
 - examples/bts_aircraft/5-2-A.htm
 - examples/bts_aircraft/5-2-B.htm
 - examples/bts_aircraft/5-2-D.htm
@@ -111,7 +139,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: fuzzy_match
-rubygems_version: 1.8.15
+rubygems_version: 1.8.21
 signing_key: 
 specification_version: 3
 summary: Find a needle in a haystack using string similarity and (optionally) regexp

data/History.txt

@@ -1,13 +0,0 @@
-== 1.3.2 / 2012-02-24
-
-* Start keeping a changelog!
-
-* Enhancements
-
-  * renamed blockings to groupings
-  * cleaned up tests
-
-* Bug fixes
-
-  * better handling for one-letter similiarities like 'X foo' vs 'X bar' which couldn't be detected by pair distance
-  * take deprecated option :tighteners as :normalizers