match-mate 0.0.3

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: fd432f4a83f343c07dceb78d41868c5a7af99f0a4d189ada8ae4f2960715eb44
+  data.tar.gz: a6ce70620c52d58d6a381013fc3157da0cf1fef5603de689abe602d1652d30a8
+SHA512:
+  metadata.gz: '08dd5ec6915e9e2fc37580551b1b126e11bd73717fce751d640957aeede69b4c8af61b83ccd82ab0971d5d2461154ff3ee038d96c2c47cf344181204d687b734'
+  data.tar.gz: 25d67e04905e242b96047c949d034516cc7b7af4eb88d9c35e357e75c5c7c61d7ce4d447e75ec25afbbf0b95efc9b3c7db1979b1aeb7b96e65c43ac4f8738dc6

data/.gitignore

@@ -0,0 +1,4 @@
+*~
+Gemfile.lock
+*.gem
+.DS_Store

data/.rspec

@@ -0,0 +1,2 @@
+--color
+--require spec_helper

data/Gemfile

@@ -0,0 +1,11 @@
+source 'https://rubygems.org'
+
+gemspec
+
+group :test do
+  gem 'pry'
+  gem 'pry-byebug'
+
+  gem 'rake'
+  gem 'rspec', '~> 3.0'
+end

data/README.md

@@ -0,0 +1,118 @@
+# match-mate
+
+A Ruby gem for matching demographics data.
+
+## Installation
+
+* * *
+
+### Prerequisites
+
+#### Python / FuzzyWuzzy
+
+match-mate uses [FuzzyWuzzy](https://github.com/seatgeek/fuzzywuzzy), a Python library that uses Levenshtein Distance to calculate the differences between sequences. It has some features not currently found in any Ruby gems, particularily the ability to ignore word order and duplicated words.
+Thus, it currently requires Python version 2.7 or higher, as well as
+
+To install FuzzyWuzzy:
+
+**On Ubuntu/Debian**
+
+    sudo apt install python python-pip
+    pip install fuzzywuzzy[speedup]
+
+**On CentOS/RHEL**
+
+    sudo yum install python python-pip
+    pip install fuzzywuzzy[speedup]
+
+**On MacOS**
+
+Python comes pre-installed on MacOS.
+
+    pip install fuzzywuzzy[speedup]
+
+#### libpostal
+
+match-mate uses [ruby_postal](https://github.com/openvenues/ruby_postal), a gem that provides Ruby bindings to [libpostal](https://github.com/openvenues/libpostal) for fast street address parsing and normalization.
+
+Before you install, make sure you have the following prerequisites:
+
+**On Ubuntu/Debian**
+
+    sudo apt-get install curl autoconf automake libtool pkg-config
+
+**On CentOS/RHEL**
+
+    sudo yum install curl autoconf automake libtool pkgconfig
+
+**On MacOS**
+
+    brew install curl autoconf automake libtool pkg-config
+
+Then to install the C library:
+
+    git clone https://github.com/openvenues/libpostal
+    cd libpostal
+    ./bootstrap.sh
+    ./configure --datadir=[...some dir with a few GB of space...]
+    make -j4
+    sudo make install
+
+    # On Linux it's probably a good idea to run
+    sudo ldconfig
+
+### Install the Gem
+
+To install using Bundler:
+
+```ruby
+gem 'match-mate', github: 'combinaut/match-mate'
+```
+
+## Configuration
+
+* * *
+
+Create an initializer and configure your Python path and match weights
+
+```ruby
+MatchMate.configure do |config|
+  config.python_path = '/usr/local/bin/python'
+  config.address_weights =  {
+    road: {
+      weight: 30,
+      threshold: 80
+    },
+    unit: {
+      weight: 2,
+      threshold: 80
+    },
+    postcode: {
+      weight: 30,
+      threshold: 100
+    },
+    city: {
+      weight: 8,
+      threshold: 80
+    },
+    house_number: {
+      weight: 30,
+      threshold: 100
+    }
+  }
+end
+```
+
+## Usage
+
+### Address Matcher
+
+Compare two addresses and get similarity score:
+
+```ruby
+address = MatchMate::Address.new "742 Evergreen Terrace Springfield IL 62704"
+other_address = MatchMate::Address.new "742 Evergreen Springfield IL 62704"
+matcher = MatchMate::AddressMatcher.new(address, other_address)
+matcher.score
+# => 100
+```

data/Rakefile

@@ -0,0 +1,10 @@
+require 'bundler'
+Bundler.setup
+Bundler::GemHelper.install_tasks
+
+require 'rspec/core/rake_task'
+desc 'Run all specs'
+RSpec::Core::RakeTask.new(:spec)
+
+task default: :spec
+task test: :spec

data/lib/match-mate.rb

@@ -0,0 +1,12 @@
+require 'active_support'
+require 'active_support/core_ext'
+
+require 'pathname'
+require 'pycall'
+require 'ruby_postal/expand'
+require 'ruby_postal/parser'
+require 'match-mate/configuration'
+require 'match-mate/utils/fuzzy_wuzzy'
+require 'match-mate/address'
+require 'match-mate/address_matcher'
+require 'match-mate/address_match_list'

data/lib/match-mate/address.rb

@@ -0,0 +1,95 @@
+module MatchMate
+  class Address
+    def self.languages
+      @languages ||= MatchMate.config.languages || DEFAULTS[:languages]
+    end
+
+    def initialize(input_object_or_string)
+      # Accepts any address object that responds to a to_s method
+      @input_string = input_object_or_string.to_s
+    end
+
+    # Basic Components for comparsion
+    def road
+      expand components[:road]
+    end
+
+    def unit
+      extract_integers components[:unit]
+    end
+
+    def postcode
+      limit_to components[:postcode], MatchMate.config.postcode_limit
+    end
+
+    def city
+      components[:city]
+    end
+
+    def state
+      components[:state]
+    end
+
+    def house_number
+      components[:house_number]
+    end
+
+    def ==(other_address)
+      AddressMatcher.new(self, other_address).match?
+    end
+
+    alias eql? ==
+    alias === ==
+
+    # Normalized Compositites
+
+    def city_and_state
+      [city, state].compact.join(', ')
+    end
+
+    def street_address
+      [house_number, road.last].compact.join(' ')
+    end
+
+    def street_adress_with_unit
+      [street_address, unit].compact.join(' ')
+    end
+
+    def to_s
+      [street_adress_with_unit, city_and_state, zip_code].compact.join(', ')
+    end
+
+    private
+
+    def extract_integers(string, limit = nil)
+      return if string.blank?
+
+      value = string.gsub(/\D/, '')
+      limit ? value.first(limit) : value
+    end
+
+    def components
+      {}.tap do |components|
+        parsed_object.each do |component|
+          components[component[:label]] = component[:value]
+        end
+      end
+    end
+
+    def parsed_object
+      @parsed_object ||= Postal::Parser.parse_address @input_string, languages: self.class.languages
+    end
+
+    def limit_to(string, limit)
+      return string if string.blank? || limit.blank?
+
+      string.first(limit)
+    end
+
+    def expand(string)
+      return if string.blank?
+
+      Postal::Expand.expand_address string, languages: self.class.languages
+    end
+  end
+end

data/lib/match-mate/address_match_list.rb

@@ -0,0 +1,39 @@
+module MatchMate
+  class AddressMatchList
+    include Enumerable
+
+    def initialize(address_object_or_string, list = [])
+      @address = address_object_or_string
+      @list = list
+    end
+
+    def each(&block)
+      scored_list.sort.each(&block)
+    end
+
+    def scored_list
+      @scored_list ||= @list.map { |address_object_or_string| AddressMatchListItem.new address_object_or_string, @address }
+    end
+  end
+
+  class AddressMatchListItem
+    include Comparable
+
+    attr_reader :address
+    delegate :score, :match?, to: :matcher
+
+    def initialize(address_object_or_string, other_address)
+      # Accepts any address object that responds to a to_s method
+      @address = address_object_or_string
+      @other_address = other_address
+    end
+
+    def matcher
+      @matcher ||= AddressMatcher.new(@address, @other_address)
+    end
+
+    def <=>(other)
+      other.score <=> score
+    end
+  end
+end

data/lib/match-mate/address_matcher.rb

@@ -0,0 +1,52 @@
+module MatchMate
+  class AddressMatcher
+    def self.attributes
+      @attributes ||= (MatchMate.config.address_weights || {}).deep_merge DEFAULTS[:address_weights]
+    end
+
+    def self.threshold
+      @threshold ||= MatchMate.config.address_threshold || DEFAULTS[:address_threshold]
+    end
+
+    def self.total_weight
+      @total_weight ||= attributes.values.sum { |config| config[:weight] }
+    end
+
+    def initialize(address, other_address)
+      @address = address.is_a?(MatchMate::Address) ? address : Address.new(address)
+      @other_address = other_address.is_a?(MatchMate::Address) ? other_address : Address.new(other_address)
+    end
+
+    def score
+      @score ||= calculate_score.round
+    end
+
+    def match?
+      score >= self.class.threshold
+    end
+
+    private
+
+    def calculate_score
+      self.class.attributes.keys.sum do |attribute|
+        weighted_score_for attribute
+      end
+    end
+
+    def weighted_score_for(attribute)
+      value = score_for(attribute)
+      config = self.class.attributes[attribute]
+      return 0 if value < config[:threshold]
+
+      value * adjusted_weight_for(config[:weight])
+    end
+
+    def adjusted_weight_for(weight)
+      ((self.class.total_weight.to_f / 100) * weight) / 100
+    end
+
+    def score_for(attribute)
+      FuzzyWuzzy.new(@address.send(attribute), @other_address.send(attribute)).token_set_ratio
+    end
+  end
+end

data/lib/match-mate/configuration.rb

@@ -0,0 +1,40 @@
+module MatchMate
+  DEFAULTS = {
+    languages: ['en'],
+    address_threshold: 80,
+    address_weights: {
+      road: {
+        weight: 30,
+        threshold: 80
+      },
+      unit: {
+        weight: 2,
+        threshold: 80
+      },
+      postcode: {
+        weight: 30,
+        threshold: 100
+      },
+      city: {
+        weight: 8,
+        threshold: 80
+      },
+      house_number: {
+        weight: 30,
+        threshold: 100
+      }
+    }
+  }.freeze
+
+  def self.configure
+    yield config
+  end
+
+  def self.config
+    @config ||= Configuration.new
+  end
+
+  class Configuration
+    attr_accessor :python_path, :postcode_limit, :address_weights, :address_threshold, :languages
+  end
+end

data/lib/match-mate/utils/fuzzy_wuzzy.rb

@@ -0,0 +1,71 @@
+PyCall.init MatchMate.config.python_path || `which python`.chomp
+
+class FuzzyWuzzy
+  def initialize(value, other_value)
+    @values = [value].flatten.map(&:to_s)
+    @other_values = [other_value].flatten.map(&:to_s)
+    PyCall.exec import
+  end
+
+  # Compares the entire string similarity, ignores word order and duplicated words
+  def token_set_ratio
+    run_command :token_set_ratio
+  end
+
+  # Compares the entire string similarity, ignores word order
+  def token_sort_ratio
+    run_command :token_sort_ratio
+  end
+
+  # Compares partial string similarity
+  def partial_ratio
+    run_command :partial_ratio
+  end
+
+  # Compares the entire string similarity, in order.
+  def ratio
+    run_command :ratio
+  end
+
+  private
+
+  def run_command(method)
+    scores = []
+    combinations.each do |value, other_value|
+      score = score_for method, value, other_value
+      scores << score
+      break if score == 100
+    end
+    scores.max
+  end
+
+  def score_for(method, value, other_value)
+    value = normalize(value)
+    other_value = normalize(other_value)
+    if value == other_value
+      100
+    else
+      PyCall.eval command(method, value, other_value)
+    end
+  end
+
+  def combinations
+    @values.product @other_values
+  end
+
+  def import
+    <<~PYTHON
+      from fuzzywuzzy import fuzz
+    PYTHON
+  end
+
+  def normalize(value)
+    value.strip.squeeze(' ').downcase
+  end
+
+  def command(method, value, other_value)
+    <<~PYTHON
+      fuzz.#{method}("#{value}", "#{other_value}")
+    PYTHON
+  end
+end

data/lib/match-mate/version.rb

@@ -0,0 +1,3 @@
+module MatchMate
+  VERSION = '0.0.3'
+end

data/match-mate.gemspec

@@ -0,0 +1,25 @@
+$:.push File.expand_path("../lib", __FILE__)
+
+require "match-mate/version"
+
+Gem::Specification.new do |s|
+  s.name        = "match-mate"
+  s.version     = MatchMate::VERSION
+  s.authors	    = ['Combinaut Team']
+  s.summary     = "A Ruby gem for matching demographics"
+  s.description = "A Ruby gem for matching demographics"
+  s.homepage    = "https://github.com/combinaut/match-mate"
+  s.license     = "MIT"
+
+  s.files         = `git ls-files -z`.split("\x0")
+  s.executables   = s.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  s.test_files    = s.files.grep(%r{^(test|spec|features)/})
+  s.require_paths = ['lib']
+  s.required_ruby_version = '>= 2.0.0'
+
+  s.add_dependency('activesupport', '~> 5.0')
+  s.add_dependency('pycall', '~> 1.2.1')
+  s.add_dependency('ruby_postal', '~> 1.0.0')
+
+  s.add_development_dependency 'bundler', '>= 1.5', '< 3.0'
+end

data/spec/lib/address_matcher.rb

@@ -0,0 +1,60 @@
+module MatchMate
+  describe AddressMatcher do
+    context 'comparing equivalent addresses' do
+      let(:address) { MatchMate::Address.new '12395 El Camino Avenue Suite 120 San Diego, CA 92130' }
+      let(:other_address) { MatchMate::Address.new '12395 El Camino Ave. Ste 120 San Diego, CA 92130' }
+      subject { MatchMate::AddressMatcher.new address, other_address }
+      describe '#score' do
+        it 'is a full match of 100' do
+          expect(subject.score).to eq(100)
+        end
+        context 'with weights that dont add up to 100' do
+          around(:each) do |example|
+            address_weights = {
+              address_weights: {
+                road: {
+                  weight: 23,
+                  threshold: 80
+                },
+                unit: {
+                  weight: 13,
+                  threshold: 80
+                },
+                postcode: {
+                  weight: 133,
+                  threshold: 100
+                },
+                city: {
+                  weight: 3,
+                  threshold: 80
+                },
+                house_number: {
+                  weight: 17,
+                  threshold: 100
+                }
+              }
+            }
+            with_config address_weights do
+              example.run
+            end
+          end
+          it 'is a full match of 100' do
+            expect(subject.score).to eq(100)
+          end
+        end
+      end
+    end
+    context 'matching address pair examples' do
+      let(:examples) do
+        [
+          ['12395 El Camino Real. Suite 120 San Diego, CA 92130', '12395 El Camino Real Ste 120 San Diego, CA 92130']
+        ]
+      end
+      it 'should all match' do
+        examples.each do |example|
+          expect(MatchMate::AddressMatcher.new(example[0], example[1])).to be_match
+        end
+      end
+    end
+  end
+end

data/spec/lib/address_spec.rb

@@ -0,0 +1,49 @@
+# Note that these examples rely on an existing appointment.
+# If you want to regenerate the VCR cassettes for these examples,
+# you'll need to generate a new appointment, adjust the `api_params` to match,
+# and then run the `success` example before the `already_cancelled` example.
+module MatchMate
+  describe Address do
+    context 'passing entire address as a string' do
+      subject { MatchMate::Address.new '12395 El Camino Ave Ste 120 San Diego, CA 92130-12456' }
+
+      describe '#road' do
+        it 'extracts an arrayt of expandeded, normalized, lowercase core street address' do
+          expect(subject.road).to include('el camino avenue')
+        end
+      end
+      describe '#unit' do
+        it 'only uses the number' do
+          expect(subject.unit).to eq('120')
+        end
+      end
+      describe '#house_number' do
+        it 'extracts the house number' do
+          expect(subject.house_number).to eq('12395')
+        end
+      end
+      describe '#postcode' do
+        context 'when postcode_limit is not set' do
+          it 'returns the full postcode' do
+            expect(subject.postcode).to eq('92130-12456')
+          end
+        end
+        context 'when postcode_limit is set' do
+          around(:each) do |example|
+            with_config(postcode_limit: 5) do
+              example.run
+            end
+          end
+          it 'extracts the first digits set by limit' do
+            expect(subject.postcode).to eq('92130')
+          end
+        end
+      end
+      describe '#city' do
+        it 'extracts lowercase city name' do
+          expect(subject.city).to eq('san diego')
+        end
+      end
+    end
+  end
+end

data/spec/lib/fuzz_wuzzy_spec.rb

@@ -0,0 +1,80 @@
+describe FuzzyWuzzy do
+  context 'comparing single strings' do
+    describe '#token_set_ratio' do
+      context 'literal match' do
+        subject { FuzzyWuzzy.new 'string', 'string' }
+        it 'returns 100' do
+          expect(subject.token_set_ratio).to eq(100)
+        end
+      end
+      context 'comparing two nil values' do
+        subject { FuzzyWuzzy.new nil, nil}
+        it 'returns 100' do
+          expect(subject.token_set_ratio).to eq(100)
+        end
+      end
+      context 'literal match ignores capitalization and spaces' do
+        subject { FuzzyWuzzy.new 'A  string   ', 'a stRing' }
+        it 'returns 100' do
+          expect(subject.token_set_ratio).to eq(100)
+        end
+      end
+      context 'close match' do
+        subject { FuzzyWuzzy.new 'swing', 'string' }
+        it 'returns less than 100' do
+          expect(subject.token_set_ratio).to be < 100
+        end
+      end
+      context 'total mismatch' do
+        subject { FuzzyWuzzy.new 'mash', 'pot' }
+        it 'returns 0' do
+          expect(subject.token_set_ratio).to eq(0)
+        end
+      end
+    end
+  end
+  context 'comparing string against array of strings' do
+    describe '#token_set_ratio' do
+      context 'full match on one of the members' do
+        subject { FuzzyWuzzy.new 'string', %w[other_string string] }
+        it 'returns 100' do
+          expect(subject.token_set_ratio).to eq(100)
+        end
+      end
+      context 'close match on one of the members' do
+        subject { FuzzyWuzzy.new 'string', %w[swing pot] }
+        it 'returns less than 100' do
+          expect(subject.token_set_ratio).to be < 100
+        end
+      end
+      context 'total mismatch on all of the members' do
+        subject { FuzzyWuzzy.new 'mash', %w[pot not] }
+        it 'returns 0' do
+          expect(subject.token_set_ratio).to eq(0)
+        end
+      end
+    end
+  end
+  context 'comparing two arrays of strings' do
+    describe '#token_set_ratio' do
+      context 'full match on one of the members' do
+        subject { FuzzyWuzzy.new ['a string', 'something'], ['other_string', 'a string'] }
+        it 'returns 100' do
+          expect(subject.token_set_ratio).to eq(100)
+        end
+      end
+      context 'close match on one of the members' do
+        subject { FuzzyWuzzy.new %w[string rack], %w[swing pot] }
+        it 'returns less than 100' do
+          expect(subject.token_set_ratio).to be < 100
+        end
+      end
+      context 'total mismatch on all of the members' do
+        subject { FuzzyWuzzy.new %w[mash rack], %w[pot not] }
+        it 'returns 0' do
+          expect(subject.token_set_ratio).to eq(0)
+        end
+      end
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,7 @@
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+
+require 'bundler'
+Bundler.setup
+Bundler.require :default, :test
+
+Dir.glob(File.expand_path('../support/**/*.rb', __FILE__), &method(:require))

data/spec/support/helpers.rb

@@ -0,0 +1,14 @@
+def with_config(options = {})
+  old_options = {}
+
+  options.each do |key, value|
+    old_options[key] = MatchMate.config.send(key)
+    MatchMate.config.send("#{key}=", value)
+  end
+
+  yield
+ensure
+  old_options.each do |key, value|
+    MatchMate.config.send("#{key}=", value)
+  end
+end

metadata

@@ -0,0 +1,128 @@
+--- !ruby/object:Gem::Specification
+name: match-mate
+version: !ruby/object:Gem::Version
+  version: 0.0.3
+platform: ruby
+authors:
+- Combinaut Team
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2020-01-21 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: activesupport
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.0'
+- !ruby/object:Gem::Dependency
+  name: pycall
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.2.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.2.1
+- !ruby/object:Gem::Dependency
+  name: ruby_postal
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.0.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.0.0
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.5'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.5'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+description: A Ruby gem for matching demographics
+email: 
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".rspec"
+- Gemfile
+- README.md
+- Rakefile
+- lib/match-mate.rb
+- lib/match-mate/address.rb
+- lib/match-mate/address_match_list.rb
+- lib/match-mate/address_matcher.rb
+- lib/match-mate/configuration.rb
+- lib/match-mate/utils/fuzzy_wuzzy.rb
+- lib/match-mate/version.rb
+- match-mate.gemspec
+- spec/lib/address_matcher.rb
+- spec/lib/address_spec.rb
+- spec/lib/fuzz_wuzzy_spec.rb
+- spec/spec_helper.rb
+- spec/support/helpers.rb
+homepage: https://github.com/combinaut/match-mate
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 2.0.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.7.6.2
+signing_key: 
+specification_version: 4
+summary: A Ruby gem for matching demographics
+test_files:
+- spec/lib/address_matcher.rb
+- spec/lib/address_spec.rb
+- spec/lib/fuzz_wuzzy_spec.rb
+- spec/spec_helper.rb
+- spec/support/helpers.rb