atlas_engine 0.1.2 → 0.2.0

data/app/countries/atlas_engine/bm/address_importer/corrections/open_address/city_alias_corrector.rb

@@ -11,22 +11,23 @@ module AtlasEngine
               extend T::Sig
 
               BM_PARISH_AND_CITY_NAMES = {
-                "Devonshire" => "Devonshire Parish",
-                "Hamilton" => "Hamilton Parish",
-                "Paget" => "Paget Parish",
-                "Pembroke" => "Pembroke Parish",
-                "Sandys" => "Sandys Parish",
-                "Smiths" => "Smiths Parish",
-                "Southampton" => "Southampton Parish",
-                "St. George's" => "St. George's Parish",
-                "Town of St. George" => "St. George",
-                "Warwick" => "Warwick Parish",
+                "City of Hamilton" => ["City of Hamilton", "Hamilton"],
+                "Devonshire" => ["Devonshire Parish", "Devonshire"],
+                "Hamilton" => ["Hamilton Parish", "Hamilton"],
+                "Paget" => ["Paget Parish", "Paget"],
+                "Pembroke" => ["Pembroke Parish", "Pembroke"],
+                "Sandys" => ["Sandys Parish", "Sandys"],
+                "Smiths" => ["Smiths Parish", "Smiths"],
+                "Southampton" => ["Southampton Parish", "Southampton"],
+                "St. George's" => ["St. George's Parish", "St. George's"],
+                "Town of St. George" => ["Town of St. George", "St. George"],
+                "Warwick" => ["Warwick Parish", "Warwick"],
               }.freeze
 
               sig { params(address: Hash).void }
               def apply(address)
                 if BM_PARISH_AND_CITY_NAMES.key?(address[:city][0])
-                  address[:city] << BM_PARISH_AND_CITY_NAMES[address[:city][0]]
+                  address[:city] = BM_PARISH_AND_CITY_NAMES[address[:city][0]]
                 end
               end
             end

data/app/countries/atlas_engine/bm/synonyms.yml

@@ -0,0 +1,6 @@
+street_synonyms:
+- lane, ln
+- road, rd
+- street, st
+- avenue, ave
+- drive, drv, dr

data/app/countries/atlas_engine/ch/country_profile.yml

@@ -2,6 +2,8 @@ id: CH
 validation:
   enabled: true
   default_matching_strategy: local
+  has_provinces: false
+  address_parser: AtlasEngine::De::ValidationTranscriber::AddressParser
   index_locales:
     - de
     - fr

data/app/countries/atlas_engine/ch/locales/de/country_profile.yml

@@ -2,7 +2,6 @@ id: CH_DE
 ingestion:
   data_mapper: AtlasEngine::AddressValidation::Es::DataMappers::DecompoundingDataMapper
 validation:
-  address_parser: AtlasEngine::De::ValidationTranscriber::AddressParser
   normalized_components:
   - region2
   - region3

data/app/countries/atlas_engine/ch/locales/fr/country_profile.yml

@@ -0,0 +1,3 @@
+id: CH_FR
+validation:
+  address_parser: AtlasEngine::Ch::Locales::Fr::ValidationTranscriber::AddressParser

data/app/countries/atlas_engine/ch/locales/fr/validation_transcriber/address_parser.rb

@@ -0,0 +1,29 @@
+# typed: true
+# frozen_string_literal: true
+
+# French address in Switzerland can be written in the following ways:
+# 1. thoroughfare type[ ]Thoroughfare name[ ]number
+# 2. number[ ]thoroughfare type[ ]Thoroughfare name
+
+module AtlasEngine
+  module Ch
+    module Locales
+      module Fr
+        module ValidationTranscriber
+          class AddressParser < AtlasEngine::ValidationTranscriber::AddressParserBase
+            private
+
+            sig { returns(T::Array[Regexp]) }
+            def country_regex_formats
+              @country_regex_formats ||=
+                [
+                  /^#{BUILDING_NUM}?\s*#{STREET_NO_COMMAS}$/o,
+                  /^#{STREET_NO_COMMAS}?\s*#{BUILDING_NUM}$/o,
+                ]
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/app/countries/atlas_engine/cz/address_validation/es/query_builder.rb

@@ -0,0 +1,43 @@
+# typed: true
+# frozen_string_literal: true
+
+module AtlasEngine
+  module Cz
+    module AddressValidation
+      module Es
+        class QueryBuilder < AtlasEngine::AddressValidation::Es::DefaultQueryBuilder
+          private
+
+          sig { returns(Hash) }
+          def street_clause
+            street_queries = [build_street_queries, empty_street_clause]
+
+            {
+              "dis_max" => {
+                "queries" => street_queries.flatten,
+              },
+            }
+          end
+
+          sig { returns(Hash) }
+          def empty_street_clause
+            {
+              "bool" => {
+                "must_not" => {
+                  "exists" => {
+                    "field" => "street",
+                  },
+                },
+              },
+            }
+          end
+
+          sig { returns(T::Array[String]) }
+          def street_query_values
+            street_names.presence || []
+          end
+        end
+      end
+    end
+  end
+end

data/app/countries/atlas_engine/cz/country_profile.yml

@@ -2,5 +2,6 @@ id: CZ
 validation:
   enabled: true
   default_matching_strategy: es
-  address_parser: AtlasEngine::Dk::ValidationTranscriber::AddressParser
+  address_parser: AtlasEngine::Cz::ValidationTranscriber::AddressParser
+  query_builder: AtlasEngine::Cz::AddressValidation::Es::QueryBuilder
   has_provinces: false

data/app/countries/atlas_engine/cz/validation_transcriber/address_parser.rb

@@ -0,0 +1,26 @@
+# typed: true
+# frozen_string_literal: true
+
+module AtlasEngine
+  module Cz
+    module ValidationTranscriber
+      class AddressParser < AtlasEngine::ValidationTranscriber::AddressParserBase
+        private
+
+        sig { returns(T::Array[Regexp]) }
+        def country_regex_formats
+          @country_regex_formats ||= [
+            /^#{BUILDING_NUM}$/,
+            /^#{STREET_NO_COMMAS}\s+#{BUILDING_NUM}?$/,
+          ]
+        end
+
+        def ridiculous?(captures, address)
+          return false if captures[:street].blank?
+
+          address.city == captures[:street]
+        end
+      end
+    end
+  end
+end

data/app/countries/atlas_engine/it/address_importer/open_address/mapper.rb

@@ -29,7 +29,7 @@ module AtlasEngine
               city: [city.titleize],
               suburb: nil,
               zip: postcode,
-              street: street.titleize,
+              street: street,
               building_and_unit_ranges: housenumber_and_unit(number, unit),
               latitude: geometry(feature)&.at(1),
               longitude: geometry(feature)&.at(0),

data/app/countries/atlas_engine/it/country_profile.yml

@@ -2,6 +2,7 @@ id: IT
 validation:
   enabled: true
   default_matching_strategy: es
+  address_parser: AtlasEngine::Dk::ValidationTranscriber::AddressParser
 ingestion:
   open_address:
     feature_mapper: AtlasEngine::It::AddressImporter::OpenAddress::Mapper

data/app/countries/atlas_engine/sa/country_profile.yml

@@ -3,7 +3,10 @@ validation:
   enabled: false
   default_matching_strategy: es
   has_provinces: false
-  restrictions:
+  normalized_components:
+  - region1
+  - city_aliases.alias
+  - street
   restrictions:
     - class: AtlasEngine::Restrictions::UnsupportedScript
       params:

data/app/countries/atlas_engine/us/country_profile.yml

@@ -2,8 +2,6 @@ id: US
 validation:
   enabled: true
   default_matching_strategy: es_street
-  city_fields:
-    - city_aliases
   address_parser: AtlasEngine::ValidationTranscriber::AddressParserNorthAmerica
 ingestion:
   settings:

data/app/lib/atlas_engine/validation_transcriber/address_parsings.rb

@@ -42,7 +42,7 @@ module AtlasEngine
 
       sig { returns(T::Array[String]) }
       def potential_building_numbers
-        parsings.pluck(:building_num).compact.uniq
+        parsings.pluck(:building_num).compact.uniq.reject { |n| n.match?(/\d{7,}/) }
       end
 
       sig { params(address_input: AddressValidation::AbstractAddress).void }

data/app/lib/atlas_engine/validation_transcriber/formatter.rb

@@ -13,8 +13,8 @@ module AtlasEngine
 
       sig { params(haystack: String, needle: String).returns(String) }
       def strip_word(haystack, needle)
-        string = haystack.sub(/[\s](#{Regexp.escape(needle)})([\s]|$)/i, " ").strip
-        string = string.sub(/[\s,](#{Regexp.escape(needle)})([\s,]|$)/i, "").strip
+        string = haystack.sub(/([\s]|^)(#{Regexp.escape(needle)})([\s]|$)/i, " ").strip
+        string = string.sub(/([\s,]|^)(#{Regexp.escape(needle)})([\s,]|$)/i, "").strip
         string = strip_trailing_punctuation(string)
         string || ""
       end

data/app/models/atlas_engine/address_validation/datastore_base.rb

@@ -20,6 +20,9 @@ module AtlasEngine
       sig { abstract.returns(T::Hash[T.untyped, T.untyped]) }
       def validation_response; end
 
+      sig { abstract.returns(CountryProfile) }
+      def country_profile; end
+
       sig { abstract.returns(ValidationTranscriber::AddressParsings) }
       def parsings; end
     end

data/app/models/atlas_engine/address_validation/es/datastore.rb

@@ -10,7 +10,12 @@ module AtlasEngine
         include DatastoreBase
         extend T::Sig
 
+        sig { override.returns(CountryProfile) }
+        attr_reader :country_profile
+
+        sig { override.returns(ValidationTranscriber::AddressParsings) }
         attr_reader :parsings
+
         attr_writer :candidates # meant for test setup only
 
         sig { params(address: AbstractAddress, locale: T.nilable(String)).void }
@@ -21,9 +26,9 @@ module AtlasEngine
           raise ArgumentError, "address has no country_code" if address.country_code.blank?
 
           @country_code = T.must(address.country_code.to_s)
-          @profile = CountryProfile.for(country_code.to_s.upcase)
+          @country_profile = CountryProfile.for(country_code.to_s.upcase, @locale)
 
-          if locale.nil? && @profile.validation.multi_locale?
+          if locale.nil? && @country_profile.validation.multi_locale?
             raise ArgumentError, "#{country_code} is a multi-locale country and requires a locale"
           end
 
@@ -107,7 +112,7 @@ module AtlasEngine
 
         private
 
-        attr_reader :address, :country_code, :locale, :profile, :query_builder
+        attr_reader :address, :country_code, :locale, :query_builder
 
         sig { returns(Token::Sequence) }
         def fetch_city_sequence_internal
@@ -153,7 +158,7 @@ module AtlasEngine
 
         sig { params(candidates: T::Array[Candidate]).void }
         def assign_term_vectors_to_candidates(candidates)
-          return if profile.validation.normalized_components.blank?
+          return if country_profile.validation.normalized_components.blank?
 
           candidate_term_vectors = measure_es_validation_request_time(method: "term_vectors") do
             repository.term_vectors(term_vectors_query(candidates))
@@ -167,7 +172,7 @@ module AtlasEngine
           {
             ids: candidates.map(&:id),
             parameters: {
-              fields: profile.validation.normalized_components,
+              fields: country_profile.validation.normalized_components,
               field_statistics: false,
             },
           }
@@ -219,7 +224,7 @@ module AtlasEngine
             FieldDecompounder.new(
               field: :street,
               value: street_value,
-              country_profile: profile,
+              country_profile:,
             ).call,
           )
         end

data/app/models/atlas_engine/address_validation/es/query_builder.rb

@@ -37,22 +37,15 @@ module AtlasEngine
         sig { returns(CountryProfile) }
         attr_reader :profile
 
-        sig { returns(Hash) }
+        sig { returns(T.nilable(Hash)) }
         def building_number_clause
-          potential_building_numbers = @parsings.potential_building_numbers.filter_map do |n|
-            AddressNumber.new(value: n).to_i
-          end.uniq
+          building_number_clause = approx_building_clauses
 
-          building_number_queries = [empty_approx_building_clause]
-          building_number_queries.unshift(
-            *T.unsafe(potential_building_numbers.map do |value|
-              approx_building_clause(value)
-            end),
-          ) if potential_building_numbers.any?
+          return if building_number_clause.nil?
 
           {
             "dis_max" => {
-              "queries" => building_number_queries,
+              "queries" => building_number_clause,
             },
           }
         end
@@ -68,6 +61,19 @@ module AtlasEngine
           }
         end
 
+        sig { returns(T.nilable(Array)) }
+        def approx_building_clauses
+          potential_building_numbers = @parsings.potential_building_numbers.filter_map do |n|
+            AddressNumber.new(value: n).to_i
+          end.uniq
+
+          if potential_building_numbers.any?
+            potential_building_numbers.map do |value|
+              approx_building_clause(value)
+            end
+          end
+        end
+
         sig { returns(Hash) }
         def empty_approx_building_clause
           {
@@ -85,25 +91,30 @@ module AtlasEngine
         def street_clause
           {
             "dis_max" => {
-              "queries" => street_query_values.map do |value|
-                {
-                  "match" => {
-                    "street" => { "query" => value, "fuzziness" => "auto" },
-                  },
-                }
-              end.union(
-                stripped_street_query_values.map do |value|
-                  {
-                    "match" => {
-                      "street_stripped" => { "query" => value, "fuzziness" => "auto" },
-                    },
-                  }
-                end,
-              ),
+              "queries" => build_street_queries,
             },
           }
         end
 
+        sig { returns(Array) }
+        def build_street_queries
+          street_query_values.map do |value|
+            {
+              "match" => {
+                "street" => { "query" => value, "fuzziness" => "auto" },
+              },
+            }
+          end.union(
+            stripped_street_query_values.map do |value|
+              {
+                "match" => {
+                  "street_stripped" => { "query" => value, "fuzziness" => "auto" },
+                },
+              }
+            end,
+          )
+        end
+
         sig { returns(T::Array[String]) }
         def street_query_values
           street_names.presence || [address.address1.to_s, address.address2.to_s].compact_blank.uniq
@@ -149,10 +160,10 @@ module AtlasEngine
         sig { returns(T.nilable(Hash)) }
         def province_clause
           {
-            "term" => {
-              "province_code" => { "value" => address.province_code.to_s.downcase },
+            "match" => {
+              "province_code" => { "query" => address.province_code.to_s.downcase },
             },
-          } if profile.attributes.dig("validation", "has_provinces")
+          } if profile.attributes.dig("validation", "has_provinces") && address.province_code.present?
         end
       end
     end

data/app/models/atlas_engine/address_validation/es/validators/full_address.rb

@@ -80,7 +80,7 @@ module AtlasEngine
               .returns(T.untyped)
           end
           def publish_notification(candidate_result: nil)
-            ActiveSupport::Notifications.instrument("atlas-engine.address_validation.validation_completed", {
+            ActiveSupport::Notifications.instrument("atlas_engine.address_validation.validation_completed", {
               candidate_result: candidate_result,
               result: result,
             }.compact)

data/app/models/atlas_engine/address_validation/log_emitter.rb

@@ -35,6 +35,7 @@ module AtlasEngine
         end
         data = {
           country_code: address.country_code,
+          matching_strategy: result.matching_strategy,
           formatted_address: formatted_address,
           concerns: concern_codes,
           suggestions: result.suggestions.map(&:attributes),

data/app/models/atlas_engine/address_validation/normalizer.rb

@@ -18,15 +18,6 @@ module AtlasEngine
           .gsub("æ", "ae")
           .gsub("œ", "oe")
           .gsub("  ", " ")
-          # normalizes Arabic characters
-          .tr("آ", "ا")
-          .tr("أ", "ا")
-          .tr("إ", "ا")
-          .tr("ئ", "ي")
-          .tr("ة", "ه")
-          .tr("ى", "ي")
-          # removes Arabic stretching characters and diacritics
-          .gsub(/[\u064B|\u064C|\u064D|\u064E|\u064F|\u0650|\u0651|\u0652|\u0640]/, "")
           # TODO: Strip hyphens for USPS not zip
           .gsub(/[!@%&"'*,.();:]/, "")
           .downcase

data/app/models/atlas_engine/address_validation/validators/full_address/address_comparison.rb

@@ -9,34 +9,18 @@ module AtlasEngine
           extend T::Sig
           include Comparable
 
-          attr_reader :street_comparison,
+          attr_reader :comparison_helper
+
+          delegate :street_comparison,
             :city_comparison,
-            :zip_comparison,
             :province_code_comparison,
-            :building_comparison
+            :zip_comparison,
+            :building_comparison,
+            to: :comparison_helper
 
           sig { params(address: AbstractAddress, candidate: Candidate, datastore: DatastoreBase).void }
           def initialize(address:, candidate:, datastore:)
-            @street_comparison = ComparisonHelper.street_comparison(
-              datastore: datastore,
-              candidate: candidate,
-            )
-            @city_comparison = ComparisonHelper.city_comparison(
-              datastore: datastore,
-              candidate: candidate,
-            )
-            @zip_comparison = ComparisonHelper.zip_comparison(
-              address: address,
-              candidate: candidate,
-            )
-            @province_code_comparison = ComparisonHelper.province_code_comparison(
-              address: address,
-              candidate: candidate,
-            )
-            @building_comparison = ComparisonHelper.building_comparison(
-              datastore: datastore,
-              candidate: candidate,
-            )
+            @comparison_helper = ComparisonHelper.new(address:, candidate:, datastore:)
           end
 
           sig { params(other: AddressComparison).returns(Integer) }

data/app/models/atlas_engine/address_validation/validators/full_address/candidate_result.rb

@@ -9,6 +9,8 @@ module AtlasEngine
           extend T::Sig
           include LogHelper
 
+          delegate :street_comparison, to: :address_comparison
+
           sig do
             params(
               candidate: AddressValidation::CandidateTuple,
@@ -25,8 +27,8 @@ module AtlasEngine
 
           sig { override.void }
           def update_result
-            result.candidate = candidate&.serialize
-            return if unmatched_components.empty?
+            result.candidate = candidate.serialize
+            return if unmatched_components_to_validate.empty?
 
             update_concerns_and_suggestions
             update_result_scope
@@ -39,8 +41,12 @@ module AtlasEngine
 
           private
 
+          sig { returns(Candidate) }
           attr_reader :candidate
 
+          sig { returns(AddressComparison) }
+          attr_reader :address_comparison
+
           sig { void }
           def update_concerns_and_suggestions
             if suggestable?
@@ -62,7 +68,7 @@ module AtlasEngine
 
           sig { void }
           def add_concerns_with_suggestions
-            unmatched_components.keys.each do |unmatched_component|
+            unmatched_components_to_validate.keys.each do |unmatched_component|
               field_name = unmatched_field_name(unmatched_component)
               if field_name.nil?
                 log_unknown_field_name
@@ -72,7 +78,7 @@ module AtlasEngine
               concern = ConcernBuilder.new(
                 unmatched_component: unmatched_component,
                 unmatched_field: field_name,
-                matched_components: matched_components.keys,
+                matched_components: matched_components_to_validate.keys,
                 address: session.address,
                 suggestion_ids: [suggestion.id].compact,
               ).build
@@ -87,7 +93,7 @@ module AtlasEngine
 
             @suggestion ||= SuggestionBuilder.from_comparisons(
               session.address.to_h,
-              unmatched_components,
+              unmatched_components_to_validate,
               candidate,
               unmatched_fields,
             )
@@ -103,11 +109,25 @@ module AtlasEngine
             @unmatched_components || split_matched_and_unmatched_components.second
           end
 
+          sig { returns(T::Hash[Symbol, AtlasEngine::AddressValidation::Token::Sequence::Comparison]) }
+          def matched_components_to_validate
+            matched_components.select do |k, _v|
+              components_to_validate.include?(k)
+            end
+          end
+
+          sig { returns(T::Hash[Symbol, AtlasEngine::AddressValidation::Token::Sequence::Comparison]) }
+          def unmatched_components_to_validate
+            unmatched_components.select do |k, _v|
+              components_to_validate.include?(k)
+            end
+          end
+
           sig { returns(T::Hash[Symbol, T.nilable(AtlasEngine::AddressValidation::Token::Sequence::Comparison)]) }
           def matched_and_unmatched_components
             components = {}
             @matched_and_unmatched_components ||= begin
-              components_to_validate.each do |field|
+              components_to_compare.each do |field|
                 components[field] = @address_comparison.send(:"#{field}_comparison")
               end
               components
@@ -116,7 +136,17 @@ module AtlasEngine
 
           sig { returns(T::Array[Symbol]) }
           def components_to_validate
-            ComponentsToValidate.new(session, candidate, street_comparison).run
+            relevant_components.components_to_validate
+          end
+
+          sig { returns(T::Array[Symbol]) }
+          def components_to_compare
+            relevant_components.components_to_compare
+          end
+
+          sig { returns(RelevantComponents) }
+          def relevant_components
+            @relevant_components ||= RelevantComponents.new(session, candidate, street_comparison)
           end
 
           sig do
@@ -124,24 +154,20 @@ module AtlasEngine
               T.nilable(AtlasEngine::AddressValidation::Token::Sequence::Comparison)]])
           end
           def split_matched_and_unmatched_components
+            return @matched_components, @unmatched_components if defined?(@matched_components) &&
+              defined?(@unmatched_components)
+
             @matched_components, @unmatched_components = matched_and_unmatched_components.partition do |_, comparison|
               comparison&.match?
             end.map(&:to_h)
           end
 
-          sig { returns(T.nilable(AtlasEngine::AddressValidation::Token::Sequence::Comparison)) }
-          def street_comparison
-            return @street_comparison if defined?(@street_comparison)
-
-            @street_comparison = @address_comparison.street_comparison
-          end
-
           sig { params(component: Symbol).returns(T.nilable(Symbol)) }
           def unmatched_field_name(component)
             return component unless component == :street
-            return if unmatched_components[:street].nil?
+            return if unmatched_components_to_validate[:street].nil?
 
-            original_street = T.must(unmatched_components[:street]).left_sequence.raw_value
+            original_street = T.must(unmatched_components_to_validate[:street]).left_sequence.raw_value
 
             if session.address.address1.to_s.include?(original_street)
               :address1