atlas_engine 0.2.0 → 0.3.0

data/app/countries/atlas_engine/pt/synonyms.yml

@@ -1,3 +1,14 @@
+street_synonyms:
+- rua, r
+- avenida, av
+- praca, pc, pca
+- travessa, tv
+- largo, lg, l, lgo
+- beco, bc
+- calcada, cc
+- professor, prof
+- camino, cam
+- estrada, estr
 city_synonyms:
 - GDM, Gondomar
 - GMR, Guimarães
@@ -5,3 +16,4 @@ city_synonyms:
 - VGS, Vagos
 - VN, Vila Nova De
 - VN, Vila Nova Da
+- TCS, Trancoso

data/app/countries/atlas_engine/pt/validation_transcriber/address_parser.rb

@@ -0,0 +1,75 @@
+# typed: true
+# frozen_string_literal: true
+
+module AtlasEngine
+  module Pt
+    module ValidationTranscriber
+      class AddressParser < AtlasEngine::ValidationTranscriber::AddressParserBase
+        private
+
+        STREET = "(?<street>.+)"
+        NUMBERED_STREET = "(?<street>.+\s+[0-9]+)"
+        BUILDING_NUM = "n?(?<building_num>[0-9]+[a-z]*)"
+        UNIT_NUM = "(?<unit_num>[[:alnum:]]+)"
+        DIRECTION = /\b(?<direction>esq|dir|dto|fte|e|d|f|esquerda|direito|frente|fundo|andar)\b\.?/i
+        PO_BOX = /\b(?<box_type>ap|apartado|caixa postal|cp)\s+(?<number>\d+)\b/i
+
+        sig { returns(T::Array[Regexp]) }
+        def country_regex_formats
+          @country_regex_formats ||= [
+            /^#{STREET},?\s+#{BUILDING_NUM}$/,
+            /^#{STREET},?\s+#{BUILDING_NUM},?\s.*$/,
+            /^#{NUMBERED_STREET},?\s+#{BUILDING_NUM}$/,
+            /^#{STREET},?\s+#{BUILDING_NUM}[\s,-]+#{UNIT_NUM}$/,
+            /^#{STREET},?\s+#{BUILDING_NUM}[\s,-]+#{UNIT_NUM}[\s,-]+#{DIRECTION}$/,
+            /^#{NUMBERED_STREET},?\s+#{BUILDING_NUM}[\s,-]+#{UNIT_NUM}$/,
+            /^#{NUMBERED_STREET},?\s+#{BUILDING_NUM}[\s,-]+#{UNIT_NUM}[\s,-]+#{DIRECTION}$/,
+          ]
+        end
+
+        sig { override.params(address_line: String).returns(T::Array[T.nilable(String)]) }
+        def extract_po_box(address_line)
+          po_box_match = address_line.match(PO_BOX)
+
+          if po_box_match
+            po_box = po_box_match["number"]
+            address_line = address_line.gsub(PO_BOX, "").strip.delete_suffix(",")
+          else
+            po_box = nil
+          end
+
+          [address_line, po_box]
+        end
+
+        # Return true if something's obviously wrong with this regex match
+        sig do
+          override.params(
+            captures: T::Hash[Symbol, T.nilable(String)],
+            address: ::AtlasEngine::AddressValidation::AbstractAddress,
+          ).returns(T::Boolean)
+        end
+        def ridiculous?(captures, address)
+          building_num = captures[:building_num]&.downcase
+          street = captures[:street]&.downcase
+          unit_num = captures[:unit_num]&.downcase
+
+          if street.present?
+            return true unless address.address1&.upcase&.include?(street.upcase) ||
+              address.address2&.upcase&.include?(street.upcase)
+          end
+
+          [building_num, unit_num].any? do |token|
+            po_box?(token) || street_suffix?(token)
+          end
+        end
+
+        sig { override.params(token: T.nilable(String)).returns(T::Boolean) }
+        def po_box?(token)
+          return false if token.blank?
+
+          token.match?(PO_BOX)
+        end
+      end
+    end
+  end
+end

data/app/countries/atlas_engine/si/address_importer/open_address/corrections/city_district_corrector.rb

@@ -0,0 +1,25 @@
+# typed: true
+# frozen_string_literal: true
+
+module AtlasEngine
+  module Si
+    module AddressImporter
+      module OpenAddress
+        module Corrections
+          class CityDistrictCorrector
+            class << self
+              extend T::Sig
+
+              sig { params(address: Hash).void }
+              def apply(address)
+                if address[:region4].present?
+                  address[:city] << address[:region4] if address[:city].exclude?(address[:region4])
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/app/countries/atlas_engine/si/address_importer/open_address/mapper.rb

@@ -0,0 +1,19 @@
+# typed: true
+# frozen_string_literal: true
+
+module AtlasEngine
+  module Si
+    module AddressImporter
+      module OpenAddress
+        class Mapper < AtlasEngine::AddressImporter::OpenAddress::DefaultMapper
+          sig do
+            params(feature: AtlasEngine::AddressImporter::OpenAddress::Feature).returns(T::Hash[Symbol, T.untyped])
+          end
+          def map(feature)
+            super(feature).merge(region4: feature["properties"]["district"])
+          end
+        end
+      end
+    end
+  end
+end

data/app/countries/atlas_engine/si/address_validation/exclusions/unknown_city.rb

@@ -0,0 +1,33 @@
+# typed: true
+# frozen_string_literal: true
+
+module AtlasEngine
+  module Si
+    module AddressValidation
+      module Exclusions
+        class UnknownCity < AtlasEngine::AddressValidation::Validators::FullAddress::Exclusions::ExclusionBase
+          extend T::Sig
+
+          class << self
+            sig do
+              override.params(
+                session: AtlasEngine::AddressValidation::Session,
+                candidate: AtlasEngine::AddressValidation::Candidate,
+                address_comparison: AtlasEngine::AddressValidation::Validators::FullAddress::AddressComparison,
+              ).returns(T::Boolean)
+            end
+            def apply?(session, candidate, address_comparison)
+              poor_city_match?(address_comparison)
+            end
+
+            private
+
+            def poor_city_match?(address_comparison)
+              address_comparison.city_comparison.sequence_comparison.aggregate_distance > 2
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/app/countries/atlas_engine/si/country_profile.yml

@@ -0,0 +1,17 @@
+id: SI
+ingestion:
+  open_address:
+    feature_mapper: AtlasEngine::Si::AddressImporter::OpenAddress::Mapper
+  correctors:
+    open_address:
+      - AtlasEngine::Si::AddressImporter::OpenAddress::Corrections::CityDistrictCorrector
+validation:
+  enabled: true
+  default_matching_strategy: es
+  exclusions:
+    city:
+      - AtlasEngine::Si::AddressValidation::Exclusions::UnknownCity
+  address_parser: AtlasEngine::Si::ValidationTranscriber::AddressParser
+  comparison_policies:
+    street:
+      unmatched: ignore_right_unmatched

data/app/countries/atlas_engine/si/synonyms.yml

@@ -0,0 +1,7 @@
+street_synonyms:
+## street suffixes
+- Slovenskih, slov
+- ulica, ul # street
+city_synonyms:
+- Slovenskih, slov
+- zgornje, zg

data/app/countries/atlas_engine/si/validation_transcriber/address_parser.rb

@@ -0,0 +1,52 @@
+# typed: true
+# frozen_string_literal: true
+
+module AtlasEngine
+  module Si
+    module ValidationTranscriber
+      class AddressParser < AtlasEngine::ValidationTranscriber::AddressParserBase
+        STREET = "(?<street>.+?)" # the .+ is non-greedy to allow for optional building number prefixes
+        BUILDING_NUM = "(?<building_num>[0-9]+(\s?[[:alpha:]]*))"
+        # the current OpenAddress dataset does not include unit numbers
+
+        sig { override.returns(T::Array[AddressComponents]) }
+        def parse
+          # addressses sometimes follow an abbreviation with a period and no space afterward
+          super.each do |components|
+            components[:street]&.gsub!(
+              /\A(?<prefix>.+?)(?<dot>\.)(?<non_space>\S)/i,
+              "\\k<prefix> \\k<non_space>",
+            )
+          end
+        end
+
+        private
+
+        sig { returns(T::Array[Regexp]) }
+        def country_regex_formats
+          @country_regex_formats ||= [
+            /^#{STREET}\s+#{BUILDING_NUM}$/,
+            /^#{STREET}$/,
+          ]
+        end
+
+        sig do
+          override.params(
+            captures: T::Hash[Symbol, T.nilable(String)],
+            address: AtlasEngine::AddressValidation::AbstractAddress,
+          ).returns(T::Boolean)
+        end
+        def ridiculous?(captures, address)
+          street = captures[:street]&.downcase
+
+          if street.present?
+            true unless address.address1&.downcase&.include?(street) ||
+              address.address2&.downcase&.include?(street)
+          end
+
+          false
+        end
+      end
+    end
+  end
+end

data/app/graphql/atlas_engine/schema.graphql

@@ -1167,7 +1167,7 @@ enum ValidationSupportedCountry {
   TO
 
   """
-  Turkey
+  Türkiye
   """
   TR
 

data/app/lib/atlas_engine/validation_transcriber/address_parser_base.rb

@@ -146,7 +146,7 @@ module AtlasEngine
         end
 
         street_tokens_ridiculous?(
-          street: T.must(street),
+          street: street,
           unit_type: unit_type,
           unit_num: unit_num,
           num_street_space: num_street_space,

data/app/models/atlas_engine/address_validation/concern_record.rb

@@ -53,13 +53,18 @@ module AtlasEngine
           new(
             **T.unsafe(
               {
-                result: result,
+                result: duplicate(result),
                 **result.address,
                 **context.except(:client_request_id),
               },
             ),
           )
         end
+
+        sig { params(obj: T.untyped).returns(T.untyped) }
+        def duplicate(obj)
+          Marshal.load(Marshal.dump(obj))
+        end
       end
 
       sig do

data/app/models/atlas_engine/address_validation/es/query_builder.rb

@@ -150,9 +150,14 @@ module AtlasEngine
           normalized_zip = ValidationTranscriber::ZipNormalizer.normalize(
             country_code: address.country_code, zip: address.zip,
           )
+
           {
             "match" => {
-              "zip" => { "query" => normalized_zip, "fuzziness" => "auto" },
+              "zip" => {
+                "query" => normalized_zip,
+                "fuzziness" => "auto",
+                "prefix_length" => profile.validation.zip_prefix_length,
+              },
             },
           }
         end

data/app/models/atlas_engine/address_validation/statsd_emitter.rb

@@ -45,7 +45,11 @@ module AtlasEngine
             concerns.each do |concern|
               tags.merge!(concern.attributes.slice(:code, :type))
 
-              StatsD.increment("AddressValidation.#{ending_breadcrumb}", tags: tags)
+              if concern.attributes[:code] == :address_unknown
+                StatsD.increment("AddressValidation.unknown", tags: tags.except(:component))
+              else
+                StatsD.increment("AddressValidation.#{ending_breadcrumb}", tags: tags)
+              end
             end
           end
         end
@@ -57,7 +61,7 @@ module AtlasEngine
       def component_concerns(component)
         if component.equal?(:street)
           result.concerns.select do |c|
-            c.attributes[:code] =~ /^(address1|address2|street).*/
+            c.attributes[:code] =~ /^(address1|address2|street|address_unknown).*/
           end
         elsif component.equal?(:building_number)
           result.concerns.select do |c|

data/app/models/atlas_engine/address_validation/token/sequence/comparator.rb

@@ -11,14 +11,22 @@ module AtlasEngine
           sig { returns(Sequence) }
           attr_reader :left, :right
 
+          sig { returns(T::Hash[T::Array[Token], Token::Comparison]) }
           attr_reader :comparison_cache
 
           MAX_ALLOWED_EDIT_DISTANCE_PERCENT = 0.5
 
-          sig { params(left_sequence: Sequence, right_sequence: Sequence).void }
-          def initialize(left_sequence:, right_sequence:)
+          sig do
+            params(
+              left_sequence: Sequence,
+              right_sequence: Sequence,
+              comparison_policy: ComparisonPolicy,
+            ).void
+          end
+          def initialize(left_sequence:, right_sequence:, comparison_policy: ComparisonPolicy::DEFAULT_POLICY)
             @left = left_sequence
             @right = right_sequence
+            @comparison_policy = comparison_policy
             @comparison_cache = Hash.new do |h, (l_tok, r_tok)|
               h[[l_tok, r_tok]] = AddressValidation::Token::Comparator.new(l_tok, r_tok).compare
             end
@@ -35,6 +43,9 @@ module AtlasEngine
 
           private
 
+          sig { returns(ComparisonPolicy) }
+          attr_reader :comparison_policy
+
           sig do
             params(
               left_permutations: T::Array[Token],
@@ -43,7 +54,7 @@ module AtlasEngine
           end
           def token_comparisons(left_permutations, right_permutations)
             left_permutations.product(right_permutations).map do |l_tok, r_tok|
-              comparison_cache[[l_tok, r_tok]]
+              T.must(comparison_cache[[l_tok, r_tok]])
             end
           end
 
@@ -133,7 +144,7 @@ module AtlasEngine
 
             remaining_right_tokens = remove_synonyms_at_same_position(remaining_right_tokens)
 
-            remaining_left_tokens.concat(remaining_right_tokens)
+            apply_unmatched_policy(remaining_left_tokens, remaining_right_tokens)
           end
 
           sig { params(token: Token, other_token: Token).returns(T::Boolean) }
@@ -151,6 +162,29 @@ module AtlasEngine
               end
               .values.flatten
           end
+
+          sig do
+            params(
+              left_unmatched_tokens: T::Array[Token],
+              right_unmatched_tokens: T::Array[Token],
+            ).returns(T::Array[Token])
+          end
+          def apply_unmatched_policy(left_unmatched_tokens, right_unmatched_tokens)
+            case comparison_policy.unmatched
+            when :ignore_left_unmatched
+              right_unmatched_tokens
+            when :ignore_right_unmatched
+              left_unmatched_tokens
+            when :ignore_largest_unmatched_side
+              if right_unmatched_tokens.size > left_unmatched_tokens.size
+                left_unmatched_tokens
+              else
+                right_unmatched_tokens
+              end
+            else
+              left_unmatched_tokens.concat(right_unmatched_tokens)
+            end
+          end
         end
       end
     end

data/app/models/atlas_engine/address_validation/token/sequence/comparison.rb

@@ -45,7 +45,7 @@ module AtlasEngine
             longest_subsequence = longest_subsequence_comparison <=> other.longest_subsequence_comparison
             return -1 * longest_subsequence if longest_subsequence.nonzero?
 
-            edit_distance = aggregate_edit_distance <=> other.aggregate_edit_distance
+            edit_distance = aggregate_distance <=> other.aggregate_distance
             return edit_distance if edit_distance.nonzero?
 
             prefixes = count_by_qualifier(:prefix) <=> other.count_by_qualifier(:prefix)
@@ -92,7 +92,7 @@ module AtlasEngine
 
           sig { returns(T::Boolean) }
           def match?
-            aggregate_edit_distance == 0 && unmatched_tokens.empty?
+            aggregate_distance == 0 && unmatched_tokens.empty?
           end
 
           sig { params(threshold_percent: Float).returns(T::Boolean) }
@@ -101,8 +101,8 @@ module AtlasEngine
           end
 
           sig { returns(Integer) }
-          def aggregate_edit_distance
-            token_comparisons.sum(&:edit_distance)
+          def aggregate_distance
+            token_comparisons.sum(&:edit_distance) + unmatched_tokens.map(&:value).sum(&:length)
           end
 
           sig { returns(Integer) }

data/app/models/atlas_engine/address_validation/token/sequence/comparison_policy.rb

@@ -0,0 +1,33 @@
+# typed: true
+# frozen_string_literal: true
+
+module AtlasEngine
+  module AddressValidation
+    class Token
+      class Sequence
+        class ComparisonPolicy
+          extend T::Sig
+
+          UNMATCHED_POLICIES = [
+            :retain,                        # keep all unmatched tokens in comparison
+            :ignore_left_unmatched,         # omit unmatched tokens from left sequence in comparison
+            :ignore_right_unmatched,        # omit unmatched tokens from right sequence in comparison
+            :ignore_largest_unmatched_side, # omit unmatched tokens from the side with the most unmatched tokens,
+            # omit from left in case of a tie
+          ].freeze
+
+          attr_reader :unmatched
+
+          sig { params(unmatched: Symbol).void }
+          def initialize(unmatched:)
+            raise "Unknown unmatched policy: #{unmatched}" if UNMATCHED_POLICIES.exclude?(unmatched)
+
+            @unmatched = unmatched
+          end
+
+          DEFAULT_POLICY = ComparisonPolicy.new(unmatched: :retain).freeze
+        end
+      end
+    end
+  end
+end

data/app/models/atlas_engine/address_validation/validators/full_address/address_comparison.rb

@@ -9,18 +9,13 @@ module AtlasEngine
           extend T::Sig
           include Comparable
 
-          attr_reader :comparison_helper
-
-          delegate :street_comparison,
-            :city_comparison,
-            :province_code_comparison,
-            :zip_comparison,
-            :building_comparison,
-            to: :comparison_helper
+          attr_reader :address, :candidate, :datastore
 
           sig { params(address: AbstractAddress, candidate: Candidate, datastore: DatastoreBase).void }
           def initialize(address:, candidate:, datastore:)
-            @comparison_helper = ComparisonHelper.new(address:, candidate:, datastore:)
+            @address = address
+            @candidate = candidate
+            @datastore = datastore
           end
 
           sig { params(other: AddressComparison).returns(Integer) }
@@ -42,13 +37,38 @@ module AtlasEngine
 
           sig { returns(T::Boolean) }
           def potential_match?
-            street_comparison.nil? || T.must(street_comparison).potential_match?
+            street_comparison.sequence_comparison.nil? || T.must(street_comparison.sequence_comparison).potential_match?
+          end
+
+          sig { returns(ZipComparison) }
+          def zip_comparison
+            @zip_comparison ||= field_comparison(field: :zip)
+          end
+
+          sig { returns(StreetComparison) }
+          def street_comparison
+            @street_comparison ||= field_comparison(field: :street)
+          end
+
+          sig { returns(CityComparison) }
+          def city_comparison
+            @city_comparison ||= field_comparison(field: :city)
+          end
+
+          sig { returns(ProvinceCodeComparison) }
+          def province_code_comparison
+            @province_code_comparison ||= field_comparison(field: :province_code)
+          end
+
+          sig { returns(BuildingComparison) }
+          def building_comparison
+            @building_comparison ||= field_comparison(field: :building)
           end
 
           protected
 
           sig do
-            returns(T::Array[T.any(AtlasEngine::AddressValidation::Token::Sequence::Comparison, NumberComparison)])
+            returns(T::Array[FieldComparisonBase])
           end
           def comparisons
             [
@@ -63,10 +83,10 @@ module AtlasEngine
           sig { returns(T::Array[AtlasEngine::AddressValidation::Token::Sequence::Comparison]) }
           def text_comparisons
             [
-              street_comparison,
-              city_comparison,
-              zip_comparison,
-              province_code_comparison,
+              street_comparison.sequence_comparison,
+              city_comparison.sequence_comparison,
+              zip_comparison.sequence_comparison,
+              province_code_comparison.sequence_comparison,
             ].compact_blank
           end
 
@@ -74,6 +94,12 @@ module AtlasEngine
           def merged_comparison
             @merged_comparisons ||= text_comparisons.reduce(&:merge)
           end
+
+          sig { params(field: Symbol).returns(FieldComparisonBase) }
+          def field_comparison(field:)
+            klass = CountryProfile.for(address.country_code).validation.address_comparison(field: field)
+            klass.new(address: address, candidate: candidate, datastore: datastore)
+          end
         end
       end
     end

data/app/models/atlas_engine/address_validation/validators/full_address/building_comparison.rb

@@ -0,0 +1,33 @@
+# typed: true
+# frozen_string_literal: true
+
+module AtlasEngine
+  module AddressValidation
+    module Validators
+      module FullAddress
+        class BuildingComparison < FieldComparisonBase
+          extend T::Sig
+
+          sig { override.returns(T.nilable(NumberComparison)) }
+          def sequence_comparison
+            @building_comparison ||= NumberComparison.new(
+              numbers: datastore.parsings.potential_building_numbers,
+              candidate_ranges: building_ranges_from_candidate(candidate),
+            )
+          end
+
+          private
+
+          sig { params(candidate: Candidate).returns(T::Array[AddressNumberRange]) }
+          def building_ranges_from_candidate(candidate)
+            building_and_unit_ranges = candidate.component(:building_and_unit_ranges)&.value
+            return [] if building_and_unit_ranges.blank?
+
+            building_ranges = JSON.parse(building_and_unit_ranges).keys
+            building_ranges.map { |building_range| AddressNumberRange.new(range_string: building_range) }
+          end
+        end
+      end
+    end
+  end
+end

data/app/models/atlas_engine/address_validation/validators/full_address/candidate_result.rb

@@ -61,7 +61,7 @@ module AtlasEngine
             concern = InvalidZipConcernBuilder.for(session.address, [])
             result.concerns << concern if concern
 
-            if ConcernBuilder.too_many_unmatched_components?(unmatched_components.keys)
+            if ConcernBuilder.too_many_unmatched_components?(session.address, unmatched_components.keys)
               result.concerns << UnknownAddressConcern.new(session.address)
             end
           end
@@ -128,7 +128,7 @@ module AtlasEngine
             components = {}
             @matched_and_unmatched_components ||= begin
               components_to_compare.each do |field|
-                components[field] = @address_comparison.send(:"#{field}_comparison")
+                components[field] = @address_comparison.send(:"#{field}_comparison").sequence_comparison
               end
               components
             end
@@ -146,7 +146,7 @@ module AtlasEngine
 
           sig { returns(RelevantComponents) }
           def relevant_components
-            @relevant_components ||= RelevantComponents.new(session, candidate, street_comparison)
+            @relevant_components ||= RelevantComponents.new(session, candidate, address_comparison)
           end
 
           sig do

data/app/models/atlas_engine/address_validation/validators/full_address/city_comparison.rb

@@ -0,0 +1,25 @@
+# typed: true
+# frozen_string_literal: true
+
+module AtlasEngine
+  module AddressValidation
+    module Validators
+      module FullAddress
+        class CityComparison < FieldComparisonBase
+          extend T::Sig
+
+          sig { override.returns(T.nilable(Token::Sequence::Comparison)) }
+          def sequence_comparison
+            return @city_comparison if defined?(@city_comparison)
+
+            @city_comparison = best_comparison(
+              datastore.fetch_city_sequence,
+              T.must(candidate.component(:city)).sequences,
+              field_policy(:city),
+            )
+          end
+        end
+      end
+    end
+  end
+end