atlas_engine 0.2.0 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4f78501b09c1f8bfeae5d8a5be2ff3b83d7f68f689e66294f704f3c3bcf5830b
-  data.tar.gz: d1100e782d95c89565be662da85ba69788fa457bbefcd11ccd8082c356086df2
+  metadata.gz: 99901e1ca20e5c32dd90945bbe7bd2199ad9cf0c11507524e7210b621f0a647c
+  data.tar.gz: c14b30e2629d198b5341e87190c0ccc38d016a644b9fea74e6d1bece611360b1
 SHA512:
-  metadata.gz: 7d2b8abdf8ae247fcb95b10a0e7f301deea99ee2505f8817c85a8bf10459e3cdedfd7a631738071cf018db7545ca0b2506ec518f31c1e5ba1f507a49915d2f25
-  data.tar.gz: d7b2177df1ca73d45a37993dfdea005d92fc3705ef422c0a1b7e90285825880f4d82f7ae3fff47d8f425abf3a2865636ec5bd8e546a2e60c113de19aeeba8df0
+  metadata.gz: b424ffd2640d6248aa00adfa8e1942fad3b0f954ecfeb339257eccf84e2976107151f7da4cf936f4f48eb450337582c1692e90d79b4ecf6658964cc78507b963
+  data.tar.gz: 463dbcc8c3ad8a9aab64ee1212fc9d9148b49c5ad44a51fece73a1d8185af58453c6d9fa4002df4f9acca3aa8ee0122a140575e4b6a90b67633fc4565d5659ce

data/README.md

@@ -1,7 +1,15 @@
-# Atlas Engine
+# 🌐  Atlas Engine
 
 Atlas Engine is a rails engine that provides a global end-to-end address validation API for rails apps.
 
+* [Address Validation API](#address-validation-api)
+* [Rails App Installation](#rails-app-installation)
+* [Local Development Installation](#local-development-installation)
+* [Address Data Ingestion](#address-data-ingestion)
+* [Elasticsearch Matching Strategy](#elasticsearch-matching-strategy)
+
+## Address Validation API
+
 The validation API is powered by GraphQL, an example request and explanation of the parameters and response follows:
 
 ```graphql
@@ -156,7 +164,7 @@ being an invalid zip code for the province `ON`. It also returns the human reada
 
 The validation scope excludes zip because the zip was not successfully validated.
 
-## Installation of Atlas Engine in your rails app
+## Rails App Installation
 
 ### Initial setup
 Add the engine to your gemfile
@@ -182,7 +190,7 @@ rails atlas_engine:install:migrations
 rails db:migrate
 ```
 
-## Setup Atlas Engine for contribution / local development
+## Local Development Installation
 
 This setup guide is based on a mac os development environment. Your tooling may vary.
 
@@ -275,7 +283,7 @@ Run sorbet check
 srb tc
 ```
 
-## Ingestion
+## Address Data Ingestion
 
 In order to power the more advanced validation matching strategies that provide city / state / zip and even street
 level address validation, your app must have a populated elasticsearch index per country available for `atlas_engine`

data/app/countries/atlas_engine/cz/country_profile.yml

@@ -5,3 +5,6 @@ validation:
   address_parser: AtlasEngine::Cz::ValidationTranscriber::AddressParser
   query_builder: AtlasEngine::Cz::AddressValidation::Es::QueryBuilder
   has_provinces: false
+  comparison_policies:
+    city:
+      unmatched: ignore_left_unmatched

data/app/countries/atlas_engine/es/country_profile.yml

@@ -0,0 +1,3 @@
+id: ES
+validation:
+  address_parser: AtlasEngine::Es::ValidationTranscriber::AddressParser

data/app/countries/atlas_engine/es/synonyms.yml

@@ -0,0 +1,2 @@
+street_synonyms:
+- c/, calle

data/app/countries/atlas_engine/es/validation_transcriber/address_parser.rb

@@ -0,0 +1,28 @@
+# typed: true
+# frozen_string_literal: true
+
+module AtlasEngine
+  module Es
+    module ValidationTranscriber
+      class AddressParser < AtlasEngine::ValidationTranscriber::AddressParserBase
+        private
+
+        NON_NUMERIC_STREET = "(?<street>[^[:digit:]]+)"
+        BUILDING_NUM_DESIGNATOR = /(?i)(n|n°|número)/
+        CATCH_ALL = /(?:,|\s|\s*.+)/
+
+        sig { returns(T::Array[Regexp]) }
+        def country_regex_formats
+          @country_regex_formats ||= [
+            /^#{STREET_NO_COMMAS},?\s+#{BUILDING_NUM}$/,
+            /^#{STREET_NO_COMMAS},?\s+#{BUILDING_NUM}#{CATCH_ALL}$/,
+            /^#{NON_NUMERIC_STREET},?\s+#{BUILDING_NUM}#{CATCH_ALL}$/,
+            /^#{STREET_NO_COMMAS},?\s+(#{BUILDING_NUM_DESIGNATOR}\s?)#{BUILDING_NUM}/,
+            /^#{STREET_NO_COMMAS},?\s+(#{BUILDING_NUM_DESIGNATOR}\s?)#{BUILDING_NUM}#{CATCH_ALL}$/,
+            /^#{NON_NUMERIC_STREET},?\s+(#{BUILDING_NUM_DESIGNATOR}\s?)#{BUILDING_NUM}#{CATCH_ALL}$/,
+          ]
+        end
+      end
+    end
+  end
+end

data/app/countries/atlas_engine/fo/address_importer/corrections/open_address/city_corrector.rb

@@ -0,0 +1,27 @@
+# typed: true
+# frozen_string_literal: true
+
+module AtlasEngine
+  module Fo
+    module AddressImporter
+      module Corrections
+        module OpenAddress
+          class CityCorrector
+            class << self
+              extend T::Sig
+
+              sig { params(address: Hash).void }
+              def apply(address)
+                if address[:city] == ["Nes, Eysturoy"] || address[:city] == ["Nes, Vágur"]
+                  address[:city] = ["Nes"]
+                elsif address[:city] == ["Syðradalur, Kalsoy"] || address[:city] == ["Syðradalur, Streymoy"]
+                  address[:city] = ["Syðradalur"]
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/app/countries/atlas_engine/fo/country_profile.yml

@@ -3,3 +3,7 @@ validation:
   enabled: true
   default_matching_strategy: es
   has_provinces: false
+ingestion:
+  correctors:
+    open_address:
+      - AtlasEngine::Fo::AddressImporter::Corrections::OpenAddress::CityCorrector

data/app/countries/atlas_engine/fr/country_profile.yml

@@ -11,3 +11,5 @@ validation:
   address_parser: AtlasEngine::Fr::ValidationTranscriber::AddressParser
   enabled: true
   default_matching_strategy: es
+  has_provinces: false
+  zip_prefix_length: 3

data/app/countries/atlas_engine/it/address_validation/validators/full_address/exclusions/city.rb

@@ -0,0 +1,31 @@
+# typed: true
+# frozen_string_literal: true
+
+module AtlasEngine
+  module It
+    module AddressValidation
+      module Validators
+        module FullAddress
+          module Exclusions
+            class City <
+              AtlasEngine::AddressValidation::Validators::FullAddress::Exclusions::ExclusionBase
+              class << self
+                sig do
+                  override.params(
+                    session: AtlasEngine::AddressValidation::Session,
+                    candidate: AtlasEngine::AddressValidation::Candidate,
+                    address_comparison: AtlasEngine::AddressValidation::Validators::FullAddress::AddressComparison,
+                  )
+                    .returns(T::Boolean)
+                end
+                def apply?(session, candidate, address_comparison)
+                  true
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/app/countries/atlas_engine/it/country_profile.yml

@@ -3,6 +3,10 @@ validation:
   enabled: true
   default_matching_strategy: es
   address_parser: AtlasEngine::Dk::ValidationTranscriber::AddressParser
+  exclusions:
+    city:
+      - AtlasEngine::It::AddressValidation::Validators::FullAddress::Exclusions::City
+  unmatched_components_suggestion_threshold: 1
 ingestion:
   open_address:
     feature_mapper: AtlasEngine::It::AddressImporter::OpenAddress::Mapper

data/app/countries/atlas_engine/kr/address_validation/validators/full_address/exclusions/city.rb

@@ -0,0 +1,69 @@
+# typed: true
+# frozen_string_literal: true
+
+module AtlasEngine
+  module Kr
+    module AddressValidation
+      module Validators
+        module FullAddress
+          module Exclusions
+            class City <
+              AtlasEngine::AddressValidation::Validators::FullAddress::Exclusions::ExclusionBase
+              extend T::Sig
+              class << self
+                COMPONENT_IDENTIFIER = {
+                  si: "시",
+                  gu: "구",
+                }.freeze
+
+                sig do
+                  override.params(
+                    session: AtlasEngine::AddressValidation::Session,
+                    candidate: AtlasEngine::AddressValidation::Candidate,
+                    address_comparison: AtlasEngine::AddressValidation::Validators::FullAddress::AddressComparison,
+                  )
+                    .returns(T::Boolean)
+                end
+                def apply?(session, candidate, address_comparison)
+                  candidate_si = extract_component_from_city(candidate, :si)
+                  candidate_gu = extract_component_from_city(candidate, :gu)
+
+                  (candidate_si.present? && contains_component?(:si, candidate_si, session)) ||
+                    (candidate_gu.present? && contains_component?(:gu, candidate_gu, session))
+                end
+
+                private
+
+                sig do
+                  params(
+                    candidate: AtlasEngine::AddressValidation::Candidate,
+                    component: Symbol,
+                  ).returns(T.nilable(String))
+                end
+                def extract_component_from_city(candidate, component)
+                  city = candidate.component(:city)&.value&.first
+
+                  city_parts = city.split(" ")
+                  city_parts.find do |part|
+                    part.end_with?(COMPONENT_IDENTIFIER[component])
+                  end
+                end
+
+                sig do
+                  params(
+                    type: Symbol,
+                    value: String,
+                    session: AtlasEngine::AddressValidation::Session,
+                  ).returns(T::Boolean)
+                end
+                def contains_component?(type, value, session)
+                  session.parsings.parsings.pluck(type)&.include?(value) || session.city&.include?(value)
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/app/countries/atlas_engine/kr/country_profile.yml

@@ -5,6 +5,13 @@ ingestion:
 validation:
   enabled: true
   default_matching_strategy: es
+  address_parser: AtlasEngine::Kr::ValidationTranscriber::AddressParser
+  comparison_policies:
+    city:
+      unmatched: ignore_left_unmatched
+  exclusions:
+    city:
+      - AtlasEngine::Kr::AddressValidation::Validators::FullAddress::Exclusions::City
   restrictions:
     - class: AtlasEngine::Restrictions::UnsupportedScript
       params:

data/app/countries/atlas_engine/kr/validation_transcriber/address_parser.rb

@@ -0,0 +1,36 @@
+# typed: true
+# frozen_string_literal: true
+
+module AtlasEngine
+  module Kr
+    module ValidationTranscriber
+      class AddressParser < AtlasEngine::ValidationTranscriber::AddressParserBase
+        private
+
+        PROVINCE = "(?<province>.+기|서울)"
+        GU = "(?<gu>.+구)"
+        SI = "(?<si>.+시)"
+        DONG = "(?<dong>.+동)"
+        EUP = "(?<eup>.+읍)"
+        STREET = "(?<street>\\S+)"
+        BUILDING_NUM = "(?<building_num>\\d+(^호)?)"
+        UNIT_NUM = "(?<unit_num>\\d+(^동)?)"
+
+        sig { returns(T::Array[Regexp]) }
+        def country_regex_formats
+          @country_regex_formats ||= [
+            %r{
+              (#{PROVINCE}\s+)?
+              (#{SI}\s+)?
+              (#{GU}\s+)?
+              (#{DONG}\s+)?
+              (#{EUP}\s+)?
+              (#{STREET}\s+)?
+              (#{BUILDING_NUM}(-|\s)?)?#{UNIT_NUM}?
+            }x,
+          ]
+        end
+      end
+    end
+  end
+end

data/app/countries/atlas_engine/lu/address_importer/corrections/open_address/city_corrector.rb

@@ -0,0 +1,45 @@
+# typed: true
+# frozen_string_literal: true
+
+module AtlasEngine
+  module Lu
+    module AddressImporter
+      module Corrections
+        module OpenAddress
+          class CityCorrector
+            class << self
+              extend T::Sig
+
+              # NOTE: ensure keys match the city names we have in our db
+              CITY_ALIASES = {
+                "luxembourg" => ["Lëtzebuerg"],
+                "esch-sur-alzette" => ["Esch-Uelzecht", "Esch/Alzette"],
+                "dudelange" => ["Diddeleng", "Düdelingen"],
+                "schifflange" => ["Schëffleng"],
+                "bettembourg" => ["Beetebuerg"],
+                "pétange" => ["Péiteng"],
+                "ettelbruck" => ["Ettelbréck"],
+                "diekirch" => ["Dikrech"],
+                "strassen" => ["Stroossen"],
+                "bertrange" => ["Bartreng"],
+                "belvaux" => ["Bieles"],
+                "differdange" => ["Déifferdeng"],
+                "wiltz" => ["Wolz"],
+                "grevenmacher" => ["Gréiwemaacher"],
+                "mersch" => ["Miersch"],
+                "redange/attert" => ["Redange", "Réiden", "Redange-sur-Attert"],
+              }
+
+              sig { params(address: Hash).void }
+              def apply(address)
+                city = address[:city].first.downcase
+                aliases = CITY_ALIASES[city.downcase] || []
+                address[:city] = address[:city] + aliases
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/app/countries/atlas_engine/lu/country_profile.yml

@@ -1,12 +1,15 @@
 id: LU
 validation:
   enabled: true
-  has_provinces: true
+  has_provinces: false
   default_matching_strategy: es
+  address_parser: AtlasEngine::Lu::ValidationTranscriber::AddressParser
   index_locales:
     - fr
     - lb
+  unmatched_components_suggestion_threshold: 1
 ingestion:
   correctors:
     open_address:
       - AtlasEngine::Lu::AddressImporter::Corrections::OpenAddress::LocaleCorrector
+      - AtlasEngine::Lu::AddressImporter::Corrections::OpenAddress::CityCorrector

data/app/countries/atlas_engine/lu/validation_transcriber/address_parser.rb

@@ -0,0 +1,23 @@
+# typed: true
+# frozen_string_literal: true
+
+module AtlasEngine
+  module Lu
+    module ValidationTranscriber
+      class AddressParser < AtlasEngine::ValidationTranscriber::AddressParserBase
+        private
+
+        STREET = "(?<street>.+)"
+        BUILDING_NUM = "(?<building_num>[0-9]+[[:alpha:]]*)([/-][0-9])?"
+
+        sig { returns(T::Array[Regexp]) }
+        def country_regex_formats
+          @country_regex_formats ||= [
+            /^#{STREET},?\s+#{BUILDING_NUM}/,
+            /^#{BUILDING_NUM}\s?(,\s?)?#{STREET}/,
+          ]
+        end
+      end
+    end
+  end
+end

data/app/countries/atlas_engine/pl/address_importer/corrections/open_address/city_corrector.rb

@@ -0,0 +1,25 @@
+# typed: true
+# frozen_string_literal: true
+
+module AtlasEngine
+  module Pl
+    module AddressImporter
+      module Corrections
+        module OpenAddress
+          class CityCorrector
+            class << self
+              extend T::Sig
+
+              sig { params(address: Hash).void }
+              def apply(address)
+                if address[:city] == ["Warszawa"]
+                  address[:city] << "Warsaw"
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/app/countries/atlas_engine/pl/address_importer/corrections/open_address/empty_street_corrector.rb

@@ -0,0 +1,32 @@
+# typed: true
+# frozen_string_literal: true
+
+module AtlasEngine
+  module Pl
+    module AddressImporter
+      module Corrections
+        module OpenAddress
+          class EmptyStreetCorrector
+            class << self
+              extend T::Sig
+
+              sig { params(address: Hash).void }
+              def apply(address)
+                if address[:street] == "" && address[:city].present?
+                  # Many smaller rural towns in Poland don't have street names. Mailing addresses are
+                  # often expressed as
+                  # address1: <town name> <building number>
+                  # city: <town name> OR <nearest postal town>
+                  # postal_code: <postal code>
+                  #
+                  # The OpenAddresses dataset does not currently include county/postal town info.
+                  address[:street] = Array(address[:city]).first
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/app/countries/atlas_engine/pl/address_validation/exclusions/placeholder_postal_code.rb

@@ -0,0 +1,35 @@
+# typed: true
+# frozen_string_literal: true
+
+module AtlasEngine
+  module Pl
+    module AddressValidation
+      module Exclusions
+        class PlaceholderPostalCode < AtlasEngine::AddressValidation::Validators::FullAddress::Exclusions::ExclusionBase
+          extend T::Sig
+
+          class << self
+            sig do
+              override.params(
+                session: AtlasEngine::AddressValidation::Session,
+                candidate: AtlasEngine::AddressValidation::Candidate,
+                address_comparison: AtlasEngine::AddressValidation::Validators::FullAddress::AddressComparison,
+              ).returns(T::Boolean)
+            end
+            def apply?(session, candidate, address_comparison)
+              placeholder_postal_code?(candidate)
+            end
+
+            private
+
+            sig { params(candidate: AtlasEngine::AddressValidation::Candidate).returns(T::Boolean) }
+            def placeholder_postal_code?(candidate)
+              zip_values = T.must(candidate.component(:zip)&.values)
+              zip_values.all?("00-000")
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/app/countries/atlas_engine/pl/address_validation/exclusions/rural_address.rb

@@ -0,0 +1,42 @@
+# typed: true
+# frozen_string_literal: true
+
+module AtlasEngine
+  module Pl
+    module AddressValidation
+      module Exclusions
+        class RuralAddress < AtlasEngine::AddressValidation::Validators::FullAddress::Exclusions::ExclusionBase
+          extend T::Sig
+
+          class << self
+            sig do
+              override.params(
+                session: AtlasEngine::AddressValidation::Session,
+                candidate: AtlasEngine::AddressValidation::Candidate,
+                address_comparison: AtlasEngine::AddressValidation::Validators::FullAddress::AddressComparison,
+              ).returns(T::Boolean)
+            end
+            def apply?(session, candidate, address_comparison)
+              rural_address?(candidate) && poor_city_match?(address_comparison)
+            end
+
+            private
+
+            def poor_city_match?(address_comparison)
+              address_comparison.city_comparison.sequence_comparison.aggregate_distance > 2
+            end
+
+            sig { params(candidate: AtlasEngine::AddressValidation::Candidate).returns(T::Boolean) }
+            def rural_address?(candidate)
+              return false if candidate.component(:city)&.values.blank?
+
+              street = candidate.component(:street)&.first_value
+              city_values = T.must(candidate.component(:city)&.values)
+              city_values.any?(street)
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/app/countries/atlas_engine/pl/country_profile.yml

@@ -1,5 +1,18 @@
 id: PL
+ingestion:
+  correctors:
+    open_address:
+      - AtlasEngine::Pl::AddressImporter::Corrections::OpenAddress::CityCorrector
+      - AtlasEngine::Pl::AddressImporter::Corrections::OpenAddress::EmptyStreetCorrector
 validation:
   address_parser: AtlasEngine::Pl::ValidationTranscriber::AddressParser
   enabled: true
   default_matching_strategy: es
+  exclusions:
+    city:
+      - AtlasEngine::Pl::AddressValidation::Exclusions::RuralAddress
+    zip:
+      - AtlasEngine::Pl::AddressValidation::Exclusions::PlaceholderPostalCode
+  comparison_policies:
+    street:
+      unmatched: ignore_largest_unmatched_side

data/app/countries/atlas_engine/pl/synonyms.yml

@@ -0,0 +1,13 @@
+street_synonyms:
+## street suffixes
+- aleja, al # avenue
+- osiedle, os # housing estate
+- plac, pl # square
+- ulica, ul # street
+## titles
+- kardynała, kard # cardinal (masculine)
+- święta, św # saint (feminine)
+- świętego, św # saint
+- święty, św # saint (masculine)
+city_synonyms:
+- wielkopolska, wlkp # Greater Poland

data/app/countries/atlas_engine/pl/validation_transcriber/address_parser.rb

@@ -5,14 +5,49 @@ module AtlasEngine
   module Pl
     module ValidationTranscriber
       class AddressParser < AtlasEngine::ValidationTranscriber::AddressParserBase
+        STREET = "(?<street>.+?)" # the .+ is non-greedy to allow for optional building number prefixes
+        BUILDING_NUM_PREFIX = "(?:nr.?\s+)"
+        BUILDING_NUM = "(?<building_num>[0-9]+[[:alpha:]]*)"
+        UNIT_NUM_PREFIX = "(?:\s*[/-]|\s+m.?)"
+        UNIT_NUM = "(?<unit_num>[[:alpha:]0-9]+)"
+
+        sig { override.returns(T::Array[AddressComponents]) }
+        def parse
+          # polish addressses sometimes follow an abbreviation with a period and no space afterward
+          super.each do |components|
+            components[:street]&.gsub!(
+              /\A(?<prefix>.+?)(?<dot>\.)(?<non_space>\S)/i,
+              "\\k<prefix> \\k<non_space>",
+            )
+          end
+        end
+
         private
 
         sig { returns(T::Array[Regexp]) }
         def country_regex_formats
           @country_regex_formats ||= [
-            %r{^(?<street>.+)\s+(?<building_num>[0-9][[:alpha:]0-9]*)(\s*/\s*(?<unit_num>[[:alpha:]0-9]+))?$},
+            /^#{STREET}\s+#{BUILDING_NUM_PREFIX}?#{BUILDING_NUM}(#{UNIT_NUM_PREFIX}\s*#{UNIT_NUM})?$/,
+            /^#{STREET}$/,
           ]
         end
+
+        sig do
+          override.params(
+            captures: T::Hash[Symbol, T.nilable(String)],
+            address: AtlasEngine::AddressValidation::AbstractAddress,
+          ).returns(T::Boolean)
+        end
+        def ridiculous?(captures, address)
+          street = captures[:street]&.downcase
+
+          if street.present?
+            true unless address.address1&.downcase&.include?(street) ||
+              address.address2&.downcase&.include?(street)
+          end
+
+          false
+        end
       end
     end
   end

data/app/countries/atlas_engine/pt/address_validation/validators/full_address/exclusions/zip.rb

@@ -0,0 +1,38 @@
+# typed: true
+# frozen_string_literal: true
+
+module AtlasEngine
+  module Pt
+    module AddressValidation
+      module Validators
+        module FullAddress
+          module Exclusions
+            class Zip < AtlasEngine::AddressValidation::Validators::FullAddress::Exclusions::ExclusionBase
+              class << self
+                sig do
+                  override.params(
+                    session: AtlasEngine::AddressValidation::Session,
+                    candidate: AtlasEngine::AddressValidation::Candidate,
+                    address_comparison: AtlasEngine::AddressValidation::Validators::FullAddress::AddressComparison,
+                  )
+                    .returns(T::Boolean)
+                end
+                def apply?(session, candidate, address_comparison)
+                  street_comparison_result = address_comparison.street_comparison.sequence_comparison
+                  building_comparison_result = address_comparison.building_comparison.sequence_comparison
+
+                  return true if street_comparison_result.nil? ||
+                    building_comparison_result.nil? ||
+                    T.must(building_comparison_result).candidate_ranges.empty?
+
+                  !T.must(street_comparison_result).match? ||
+                    !T.must(building_comparison_result).match?
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/app/countries/atlas_engine/pt/country_profile.yml

@@ -8,3 +8,7 @@ ingestion:
 validation:
   enabled: true
   default_matching_strategy: es
+  address_parser: AtlasEngine::Pt::ValidationTranscriber::AddressParser
+  exclusions:
+    zip:
+      - AtlasEngine::Pt::AddressValidation::Validators::FullAddress::Exclusions::Zip