atlas_engine 0.1.0

data/app/models/atlas_engine/address_validation/es/datastore.rb

@@ -0,0 +1,229 @@
+# typed: true
+# frozen_string_literal: true
+
+module AtlasEngine
+  module AddressValidation
+    module Es
+      class Datastore
+        include MetricsHelper
+        include LogHelper
+        include DatastoreBase
+        extend T::Sig
+
+        attr_reader :parsings
+        attr_writer :candidates # meant for test setup only
+
+        sig { params(address: AbstractAddress, locale: T.nilable(String)).void }
+        def initialize(address:, locale: nil)
+          @address = address
+          @locale = locale
+
+          raise ArgumentError, "address has no country_code" if address.country_code.blank?
+
+          @country_code = T.must(address.country_code.to_s)
+          @profile = CountryProfile.for(country_code.to_s.upcase)
+
+          if locale.nil? && @profile.validation.multi_locale?
+            raise ArgumentError, "#{country_code} is a multi-locale country and requires a locale"
+          end
+
+          @parsings = ValidationTranscriber::AddressParsings.new(address_input: address, locale: locale)
+          @query_builder = QueryBuilder.for(address, locale)
+        end
+
+        sig do
+          returns(CountryRepository)
+        end
+        def repository
+          @repository ||= CountryRepository.new(
+            country_code: country_code.downcase,
+            repository_class: AtlasEngine.elasticsearch_repository.constantize,
+            locale: locale&.downcase,
+            index_configuration: nil,
+          )
+        end
+
+        sig { params(sequence: Token::Sequence).void }
+        def city_sequence=(sequence)
+          @city_sequence_future = Concurrent::Promises.fulfilled_future(sequence)
+        end
+
+        sig { override.returns(Token::Sequence) }
+        def fetch_city_sequence
+          log_future_state_on_join(future: @city_sequence_future, method: "city_sequence")
+
+          @city_sequence_future ||= Concurrent::Promises.fulfilled_future(fetch_city_sequence_internal)
+
+          @city_sequence_future.value!
+        end
+
+        sig { returns(Concurrent::Promises::Future) }
+        def fetch_city_sequence_async
+          submit_time = Time.current
+
+          @city_sequence_future ||= Concurrent::Promises.future do
+            measure_future_queue_time(enqueue_time: submit_time, method: "city_sequence")
+            fetch_city_sequence_internal
+          end
+        end
+
+        sig { params(sequences: T::Array[Token::Sequence]).void }
+        def street_sequences=(sequences)
+          @street_sequences_future = Concurrent::Promises.fulfilled_future(sequences)
+        end
+
+        sig { override.returns(T::Array[Token::Sequence]) }
+        def fetch_street_sequences
+          log_future_state_on_join(future: @street_sequences_future, method: "all_street_sequences")
+
+          @street_sequences_future ||= Concurrent::Promises.fulfilled_future(fetch_street_sequences_internal)
+
+          @street_sequences_future.value!
+        end
+
+        sig { returns(Concurrent::Promises::Future) }
+        def fetch_street_sequences_async
+          submit_time = Time.current
+
+          @street_sequences_future ||= Concurrent::Promises.future do
+            measure_future_queue_time(enqueue_time: submit_time, method: "all_street_sequences")
+            fetch_street_sequences_internal
+          end
+        end
+
+        sig { override.returns(T::Array[Candidate]) }
+        def fetch_full_address_candidates
+          @candidates ||= fetch_addresses_internal.map { |address| Candidate.from(address) }.tap do |candidates|
+            assign_term_vectors_to_candidates(candidates) if candidates.present?
+          end
+        end
+
+        sig { override.returns(Hash) }
+        def validation_response
+          {
+            body: fetch_addresses_internal,
+          }
+        end
+
+        private
+
+        attr_reader :address, :country_code, :locale, :profile, :query_builder
+
+        sig { returns(Token::Sequence) }
+        def fetch_city_sequence_internal
+          city_value = address.city
+          request = {
+            analyzer: :city_analyzer,
+            text: city_value,
+          }
+
+          measure_es_validation_request_time(method: "city_sequence") do
+            tokens = repository.analyze(request).map do |token|
+              Token.from_analyze(token)
+            end
+            Token::Sequence.new(tokens: tokens, raw_value: city_value)
+          end
+        end
+
+        sig { returns(T::Array[T::Hash[String, T.untyped]]) }
+        def fetch_addresses_internal
+          measure_es_validation_request_time(method: "full_address_candidates") do
+            repository.search(query_builder.full_address_query)
+          end
+        end
+
+        sig { returns(T::Array[Token::Sequence]) }
+        def fetch_street_sequences_internal
+          measure_es_validation_request_time(method: "all_street_sequences") do
+            @parsings.potential_streets.map do |street_address_value|
+              request = {
+                analyzer: :street_analyzer,
+                text: prepare_street_for_analysis(street_address_value),
+              }
+
+              measure_es_validation_request_time(method: "street_sequence") do
+                tokens = repository.analyze(request).map do |token|
+                  Token.from_analyze(token)
+                end
+                Token::Sequence.new(tokens: tokens, raw_value: street_address_value)
+              end
+            end
+          end
+        end
+
+        sig { params(candidates: T::Array[Candidate]).void }
+        def assign_term_vectors_to_candidates(candidates)
+          return if profile.validation.normalized_components.blank?
+
+          candidate_term_vectors = measure_es_validation_request_time(method: "term_vectors") do
+            repository.term_vectors(term_vectors_query(candidates))
+          end
+
+          TermVectors.new(term_vectors_hashes: candidate_term_vectors, candidates: candidates).set_candidate_sequences
+        end
+
+        sig { params(candidates: T::Array[Candidate]).returns(T::Hash[String, T.untyped]) }
+        def term_vectors_query(candidates)
+          {
+            ids: candidates.map(&:id),
+            parameters: {
+              fields: profile.validation.normalized_components,
+              field_statistics: false,
+            },
+          }
+        end
+
+        sig { params(method: String, block: T.proc.returns(T.untyped)).returns(T.untyped) }
+        def measure_es_validation_request_time(method:, &block)
+          measure_distribution(
+            name: "AddressValidation.elasticsearch_request_time_dist",
+            tags: [
+              "country:#{country_code}",
+              "method:#{method}",
+            ],
+            &block
+          )
+        end
+
+        sig { params(future: T.nilable(Concurrent::Promises::Future), method: String).void }
+        def log_future_state_on_join(future:, method:)
+          state = future&.state || :unsubmitted
+          log_warn("Joining with #{state} future, method: #{method}") unless state == :fulfilled
+
+          StatsD.increment(
+            "AddressValidation.elasticsearch_future_state",
+            sample_rate: 1.0,
+            tags: {
+              country: country_code,
+              method:,
+              state:,
+            },
+          )
+        end
+
+        sig { params(enqueue_time: ActiveSupport::TimeWithZone, method: String).void }
+        def measure_future_queue_time(enqueue_time:, method:)
+          StatsD.distribution(
+            "AddressValidation.elasticsearch_future_queue_time",
+            Time.current - enqueue_time,
+            tags: [
+              "country:#{country_code}",
+              "method:#{method}",
+            ],
+          )
+        end
+
+        sig { params(street_value: String).returns(String) }
+        def prepare_street_for_analysis(street_value)
+          T.must(
+            FieldDecompounder.new(
+              field: :street,
+              value: street_value,
+              country_profile: profile,
+            ).call,
+          )
+        end
+      end
+    end
+  end
+end

data/app/models/atlas_engine/address_validation/es/default_query_builder.rb

@@ -0,0 +1,30 @@
+# typed: true
+# frozen_string_literal: true
+
+module AtlasEngine
+  module AddressValidation
+    module Es
+      class DefaultQueryBuilder < QueryBuilder
+        sig { override.returns(T::Hash[String, T.untyped]) }
+        def full_address_query
+          clauses = [
+            building_number_clause,
+            street_clause,
+            city_clause,
+            zip_clause,
+            province_clause,
+          ].compact
+          {
+            "query" => {
+              "bool" =>
+                {
+                  "should" => clauses,
+                  "minimum_should_match" => [clauses.count - 2, 2].max,
+                },
+            },
+          }
+        end
+      end
+    end
+  end
+end

data/app/models/atlas_engine/address_validation/es/query_builder.rb

@@ -0,0 +1,160 @@
+# typed: true
+# frozen_string_literal: true
+
+module AtlasEngine
+  module AddressValidation
+    module Es
+      class QueryBuilder
+        extend T::Helpers
+        extend T::Sig
+
+        abstract!
+        class << self
+          extend T::Sig
+
+          sig { params(address: AbstractAddress, locale: T.nilable(String)).returns(QueryBuilder) }
+          def for(address, locale = nil)
+            profile = CountryProfile.for(T.must(address.country_code), locale)
+            profile.attributes.dig("validation", "query_builder").constantize.new(address, locale)
+          end
+        end
+
+        sig { params(address: AbstractAddress, locale: T.nilable(String)).void }
+        def initialize(address, locale = nil)
+          @address = address
+          @profile = CountryProfile.for(T.must(address.country_code), locale)
+          @parsings = ValidationTranscriber::AddressParsings.new(address_input: address, locale: locale)
+        end
+
+        sig { abstract.returns(T::Hash[String, T.untyped]) }
+        def full_address_query; end
+
+        private
+
+        sig { returns(AbstractAddress) }
+        attr_reader :address
+
+        sig { returns(CountryProfile) }
+        attr_reader :profile
+
+        sig { returns(Hash) }
+        def building_number_clause
+          potential_building_numbers = @parsings.potential_building_numbers.filter_map do |n|
+            AddressNumber.new(value: n).to_i
+          end.uniq
+
+          building_number_queries = [empty_approx_building_clause]
+          building_number_queries.unshift(
+            *T.unsafe(potential_building_numbers.map do |value|
+              approx_building_clause(value)
+            end),
+          ) if potential_building_numbers.any?
+
+          {
+            "dis_max" => {
+              "queries" => building_number_queries,
+            },
+          }
+        end
+
+        sig { params(value: Integer).returns(Hash) }
+        def approx_building_clause(value)
+          {
+            "term" => {
+              "approx_building_ranges" => {
+                "value" => value,
+              },
+            },
+          }
+        end
+
+        sig { returns(Hash) }
+        def empty_approx_building_clause
+          {
+            "bool" => {
+              "must_not" => {
+                "exists" => {
+                  "field" => "approx_building_ranges",
+                },
+              },
+            },
+          }
+        end
+
+        sig { returns(Hash) }
+        def street_clause
+          {
+            "dis_max" => {
+              "queries" => street_query_values.map do |value|
+                {
+                  "match" => {
+                    "street" => { "query" => value, "fuzziness" => "auto" },
+                  },
+                }
+              end.union(
+                stripped_street_query_values.map do |value|
+                  {
+                    "match" => {
+                      "street_stripped" => { "query" => value, "fuzziness" => "auto" },
+                    },
+                  }
+                end,
+              ),
+            },
+          }
+        end
+
+        sig { returns(T::Array[String]) }
+        def street_query_values
+          street_names.presence || [address.address1.to_s, address.address2.to_s].compact_blank.uniq
+        end
+
+        sig { returns(T::Array[String]) }
+        def street_names
+          streets = @parsings.potential_streets
+          (streets + streets.map { |street| Street.new(street: street).with_stripped_name }).uniq
+        end
+
+        sig { returns(T::Array[String]) }
+        def stripped_street_query_values
+          @parsings.potential_streets.map { |street| Street.new(street: street).with_stripped_name }.uniq
+        end
+
+        sig { returns(T.nilable(Hash)) }
+        def city_clause
+          {
+            "nested" => {
+              "path" => "city_aliases",
+              "query" => {
+                "match" => {
+                  "city_aliases.alias" => { "query" => address.city.to_s, "fuzziness" => "auto" },
+                },
+              },
+            },
+          }
+        end
+
+        sig { returns(Hash) }
+        def zip_clause
+          normalized_zip = ValidationTranscriber::ZipNormalizer.normalize(
+            country_code: address.country_code, zip: address.zip,
+          )
+          {
+            "match" => {
+              "zip" => { "query" => normalized_zip, "fuzziness" => "auto" },
+            },
+          }
+        end
+
+        sig { returns(T.nilable(Hash)) }
+        def province_clause
+          {
+            "term" => {
+              "province_code" => { "value" => address.province_code.to_s.downcase },
+            },
+          } if profile.attributes.dig("validation", "has_provinces")
+        end
+      end
+    end
+  end
+end

data/app/models/atlas_engine/address_validation/es/term_vectors.rb

@@ -0,0 +1,78 @@
+# typed: true
+# frozen_string_literal: true
+
+module AtlasEngine
+  module AddressValidation
+    module Es
+      class TermVectors
+        extend T::Sig
+
+        sig do
+          params(
+            term_vectors_hashes: T::Array[T::Hash[String, T::Hash[String, T.untyped]]],
+            candidates: T::Array[Candidate],
+          ).void
+        end
+        def initialize(term_vectors_hashes:, candidates:)
+          @term_vectors_hashes = term_vectors_hashes
+          @candidates = candidates
+        end
+
+        sig { void }
+        def set_candidate_sequences
+          candidates_by_id = candidates.index_by(&:id)
+
+          term_vectors_hashes.map do |candidate_result|
+            candidate = candidates_by_id[candidate_result["_id"]]
+
+            next if candidate.nil?
+
+            candidate_result["term_vectors"].map do |component_name, terms_hash|
+              component_name = component_name.delete_suffix("_decompounded")
+              # city values are indexed as city_aliases.alias, but Atlas still uses :city as the component name
+              component_name = "city" if component_name == "city_aliases.alias"
+              component = candidate.component(component_name.to_sym)
+              sorted_tokens = Token.from_field_term_vector(terms_hash)
+              set_sequences(component, sorted_tokens)
+            end
+          end
+        end
+
+        private
+
+        attr_reader :term_vectors_hashes, :candidates
+
+        sig do
+          params(component: Candidate::Component, sorted_tokens: T::Array[Token]).void
+        end
+        def set_sequences(component, sorted_tokens)
+          grouped_tokens = split_tokens_by_position(sorted_tokens)
+          component.sequences = grouped_tokens.map.with_index do |sequence_tokens, value_index|
+            # ES' offsets are set as if all tokens are part of one long sequence
+            # we adjust the offsets to be relative to the start of each sequence
+            offset = T.must(sequence_tokens.first).start_offset
+            sequence_tokens.each_with_index do |token, i|
+              token.start_offset = token.start_offset - offset
+              token.end_offset = token.end_offset - offset
+              token.position = i
+            end
+            Token::Sequence.new(
+              tokens: sequence_tokens,
+              raw_value: component.values[value_index],
+            )
+          end
+        end
+
+        sig do
+          params(tokens: T::Array[Token])
+            .returns(T::Enumerable[T::Array[Token]])
+        end
+        def split_tokens_by_position(tokens)
+          tokens.chunk_while do |token, next_token|
+            token.preceeds?(next_token)
+          end
+        end
+      end
+    end
+  end
+end

data/app/models/atlas_engine/address_validation/es/validators/full_address.rb

@@ -0,0 +1,123 @@
+# typed: true
+# frozen_string_literal: true
+
+module AtlasEngine
+  module AddressValidation
+    module Es
+      module Validators
+        class FullAddress < FullAddressValidatorBase
+          include LogHelper
+
+          attr_reader :address, :result
+          attr_accessor :session
+
+          sig { params(address: TAddress, result: Result).void }
+          def initialize(address:, result: Result.new)
+            super
+            @session = T.let(Session.new(address: address, matching_strategy: MatchingStrategies::Es), Session)
+          end
+
+          sig { override.returns(Result) }
+          def validate
+            return result if concerns_preclude_validation
+
+            candidate_result = build_candidate_result
+            candidate_result.update_result
+            publish_notification(candidate_result: candidate_result)
+            result
+          end
+
+          sig { returns(AddressValidation::Validators::FullAddress::CandidateResultBase) }
+          def build_candidate_result
+            unless supported_address?(address)
+              return AddressValidation::Validators::FullAddress::UnsupportedScriptResult.new(session:, result:)
+            end
+
+            if best_candidate.nil?
+              AddressValidation::Validators::FullAddress::NoCandidateResult.new(session:, result:)
+            else
+              AddressValidation::Validators::FullAddress::CandidateResult.new(
+                candidate: T.must(best_candidate),
+                result: result,
+                session: session,
+              )
+            end
+          end
+
+          private
+
+          sig { returns(T.nilable(CandidateTuple)) }
+          def best_candidate
+            @best_candidate ||= T.let(
+              begin
+                index_locales = CountryProfile.for(address.country_code).validation.index_locales
+
+                candidate_futures = if index_locales.empty?
+                  [best_candidate_future]
+                else
+                  index_locales.map { |locale| best_candidate_future(locale) }
+                end
+
+                candidate_futures.filter_map(&:value!).min
+              ensure
+                # We want our futures to complete even when we do not consume their value.
+                candidate_futures&.map(&:wait!)
+              end,
+              T.nilable(CandidateTuple),
+            )
+          end
+
+          sig { params(locale: T.nilable(String)).returns(Concurrent::Promises::Future) }
+          def best_candidate_future(locale = nil)
+            AddressValidation::Es::CandidateSelector.new(
+              datastore: session.datastore(locale: locale),
+              address: session.address,
+            ).best_candidate_async
+          end
+
+          sig do
+            params(candidate_result: T.nilable(AddressValidation::Validators::FullAddress::CandidateResultBase))
+              .returns(T.untyped)
+          end
+          def publish_notification(candidate_result: nil)
+            ActiveSupport::Notifications.instrument("atlas-engine.address_validation.validation_completed", {
+              candidate_result: candidate_result,
+              result: result,
+            }.compact)
+          end
+
+          sig { returns(T::Boolean) }
+          def concerns_preclude_validation
+            has_error_concerns? || exceeds_max_token_length?
+          end
+
+          sig { returns(T::Boolean) }
+          def has_error_concerns?
+            error_concerns = result.concerns.select { |concern| concern.type == Concern::TYPES[:error] }
+            error_concerns.flat_map(&:field_names).intersect?([
+              :country,
+              :province,
+              :city,
+              :zip,
+              :address1,
+              :address2,
+            ])
+          end
+
+          sig { returns(T::Boolean) }
+          def exceeds_max_token_length?
+            result.concerns.flat_map(&:code).intersect?([
+              :address1_contains_too_many_words,
+              :address2_contains_too_many_words,
+            ])
+          end
+
+          sig { params(address: TAddress).returns(T::Boolean) }
+          def supported_address?(address)
+            RestrictionEvaluator.new(address).supported_address?
+          end
+        end
+      end
+    end
+  end
+end

data/app/models/atlas_engine/address_validation/es/validators/full_address_street.rb

@@ -0,0 +1,18 @@
+# typed: true
+# frozen_string_literal: true
+
+module AtlasEngine
+  module AddressValidation
+    module Es
+      module Validators
+        class FullAddressStreet < FullAddress
+          sig { params(address: TAddress, result: Result).void }
+          def initialize(address:, result: Result.new)
+            super
+            @session = T.let(Session.new(address: address, matching_strategy: MatchingStrategies::EsStreet), Session)
+          end
+        end
+      end
+    end
+  end
+end

data/app/models/atlas_engine/address_validation/es/validators/restriction_evaluator.rb

@@ -0,0 +1,37 @@
+# typed: true
+# frozen_string_literal: true
+
+module AtlasEngine
+  module AddressValidation
+    module Es
+      module Validators
+        class RestrictionEvaluator
+          extend T::Sig
+          attr_reader :address
+
+          sig { params(address: AtlasEngine::AddressValidation::AbstractAddress).void }
+          def initialize(address)
+            @address = address
+          end
+
+          sig { returns(T::Boolean) }
+          def supported_address?
+            country_profile = CountryProfile.for(T.must(address.country_code))
+
+            country_profile.attributes.dig("validation", "restrictions").map do |restriction|
+              class_name = restriction["class"]
+              additional_params = restriction["params"]&.transform_keys(&:to_sym)
+
+              params = { address: address }
+              params = params.merge!({ params: additional_params }) if additional_params.present?
+
+              return false if class_name.constantize.send(:apply?, **params)
+            end
+
+            true
+          end
+        end
+      end
+    end
+  end
+end