atlas_engine 0.1.0

data/app/models/atlas_engine/elasticsearch/repository.rb

@@ -0,0 +1,246 @@
+# typed: strict
+# frozen_string_literal: true
+
+module AtlasEngine
+  module Elasticsearch
+    class Repository
+      include RepositoryInterface
+      extend T::Sig
+      extend T::Helpers
+
+      INITIAL_INDEX_VERSION = 0
+
+      sig { returns(ClientInterface) }
+      attr_reader :client
+
+      sig { returns(String) }
+      attr_reader :index_base_name
+
+      sig { returns(T::Hash[String, T.untyped]) }
+      attr_reader :index_mappings
+
+      sig { returns(T::Hash[Symbol, T.untyped]) }
+      attr_reader :index_settings
+
+      sig { returns(T.proc.params(arg0: T.untyped).returns(T.untyped)) }
+      attr_reader :mapper_callable
+
+      sig do
+        override.params(
+          index_base_name: String,
+          index_settings: T::Hash[Symbol, T.untyped],
+          index_mappings: T.nilable(T::Hash[String, T.untyped]),
+          mapper_callable: T.nilable(T.proc.params(arg0: T.untyped).returns(T.untyped)),
+        ).void
+      end
+      def initialize(index_base_name:, index_settings:, index_mappings:, mapper_callable: nil)
+        @client = T.let(Client.new, ClientInterface)
+        @index_base_name = T.let(index_base_name, String)
+        @index_mappings = T.let(index_mappings || default_mapping, T::Hash[String, T.untyped])
+        @index_settings = T.let(index_settings, T::Hash[Symbol, T.untyped])
+        @mapper_callable = T.let(
+          mapper_callable || ->(record) { record.to_hash },
+          T.proc.params(arg0: T.untyped).returns(T.untyped),
+        )
+      end
+
+      sig { override.returns(String) }
+      def base_alias_name
+        if Rails.env.test?
+          "test_#{index_base_name.to_s.downcase}"
+        else
+          index_base_name.to_s.downcase
+        end
+      end
+
+      sig { override.returns(String) }
+      def read_alias_name
+        base_alias_name
+      end
+
+      sig do
+        override.params(
+          ensure_clean: T::Boolean,
+          raise_errors: T::Boolean,
+        ).void
+      end
+      def create_next_index(ensure_clean: false, raise_errors: false)
+        # PENDING: cleanup next index if ensure_clean = true
+        return if client.index_or_alias_exists?(new_alias)
+
+        versioned_index_name = if client.index_or_alias_exists?(active_alias)
+          T.must(client.find_index_by(alias_name: active_alias)).next
+        else
+          "#{active_alias}.#{INITIAL_INDEX_VERSION}"
+        end
+
+        body = {
+          aliases: {
+            new_alias.to_s => {
+              is_write_index: true,
+            },
+          },
+          settings: index_settings,
+          mappings: index_mappings,
+        }
+
+        client.put(versioned_index_name, body)
+      rescue
+        raise if raise_errors
+      end
+
+      sig do
+        override.params(
+          raise_errors: T::Boolean,
+        ).void
+      end
+      def switch_to_next_index(raise_errors: false)
+        update_all_aliases_of_index if client.index_or_alias_exists?(new_alias)
+      rescue
+        raise if raise_errors
+      end
+
+      sig do
+        override.params(
+          records: T.any(ActiveRecord::Relation, T::Array[PostAddressData]),
+        ).returns(T.nilable(Response))
+      end
+      def save_records_backfill(records)
+        return if records.blank?
+
+        alias_name = if client.index_or_alias_exists?(new_alias)
+          new_alias
+        elsif client.index_or_alias_exists?(active_alias)
+          active_alias
+        else
+          raise "Next or current index must exist to backfill records"
+        end
+
+        body = ""
+        records.each do |record|
+          body += <<-NDJSON
+            { "create": {} }
+            #{record_source(record).to_json}
+          NDJSON
+        end
+
+        client.post("/#{alias_name}/_bulk", body)
+      end
+
+      sig { override.params(query: T::Hash[String, T.untyped]).returns(T::Hash[String, T.untyped]) }
+      def search(query)
+        path = "/#{active_alias}/_search"
+        response = client.post(path, query, {})
+
+        response.body
+      rescue ::Elastic::Transport::Transport::Error
+        { "hits" => { "hits" => [] } }
+      end
+
+      sig { override.params(query: T::Hash[String, T.untyped]).returns(T::Hash[String, T.untyped]) }
+      def analyze(query)
+        path = "/#{active_alias}/_analyze"
+        response = client.post(path, query, {})
+
+        response.body
+      rescue ::Elastic::Transport::Transport::Error
+        { "tokens" => [] }
+      end
+
+      sig { override.params(query: T::Hash[String, T.untyped]).returns(T::Hash[String, T.untyped]) }
+      def term_vectors(query)
+        path = "/#{active_alias}/_mtermvectors"
+        response = client.post(path, query, {})
+
+        response.body
+      rescue ::Elastic::Transport::Transport::Error
+        { "docs" => [] }
+      end
+
+      sig { override.params(id: T.any(String, Integer)).returns(T::Hash[String, T.untyped]) }
+      def find(id)
+        path = "/#{active_alias}/_doc/#{id}"
+        response = client.get(path, nil, {})
+
+        response.body["_source"]
+      rescue ::Elastic::Transport::Transport::Error
+        {}
+      end
+
+      sig { override.params(post_address: PostAddressData).returns(T::Hash[Symbol, T.untyped]) }
+      def record_source(post_address)
+        mapper_callable.call(post_address).compact
+      end
+
+      sig { returns(T::Array[T.untyped]) }
+      def indices
+        path = "/_cat/indices?format=json"
+        response = client.get(path, nil, {})
+
+        response.body
+      end
+
+      private
+
+      sig { returns(T::Hash[String, T.untyped]) }
+      def default_mapping
+        {
+          "dynamic" => "false",
+          "properties" => {
+            "id" => { "type" => "long" },
+          },
+        }
+      end
+
+      sig { void }
+      def update_all_aliases_of_index
+        previous_index_name = client.find_index_by(alias_name: archived_alias)
+        current_index_name = client.find_index_by(alias_name: active_alias)
+        next_index_name = client.find_index_by(alias_name: new_alias)
+
+        # delete the .archive alias
+        if previous_index_name.present?
+          client.delete(previous_index_name)
+        end
+
+        # archve the current alias
+        if current_index_name.present?
+          update_aliases_for_index(
+            index_name: current_index_name,
+            remove_alias: active_alias,
+            add_alias: archived_alias,
+          )
+        end
+
+        # activate the .new alias
+        if next_index_name.present?
+          update_aliases_for_index(
+            index_name: next_index_name,
+            remove_alias: new_alias,
+            add_alias: active_alias,
+          )
+        end
+      end
+
+      sig do
+        params(
+          index_name: String,
+          remove_alias: String,
+          add_alias: String,
+        ).void
+      end
+      def update_aliases_for_index(index_name:, remove_alias:, add_alias:)
+        is_writable = add_alias == active_alias ? true : false
+
+        body = {
+          actions: [
+            { remove: { index: index_name, alias: remove_alias } },
+            { add: { index: index_name, alias: add_alias, is_write_index: is_writable } },
+          ],
+        }
+
+        client.post("/_aliases", body)
+      end
+    end
+  end
+end

data/app/models/atlas_engine/elasticsearch/repository_interface.rb

@@ -0,0 +1,82 @@
+# typed: strict
+# frozen_string_literal: true
+
+module AtlasEngine
+  module Elasticsearch
+    module RepositoryInterface
+      extend T::Sig
+      extend T::Helpers
+      abstract!
+
+      PostAddressData = T.type_alias { T.any(PostAddress, T::Hash[Symbol, T.untyped]) }
+
+      sig do
+        abstract.params(
+          index_base_name: String,
+          index_settings: T::Hash[Symbol, T.untyped],
+          index_mappings: T.nilable(T::Hash[String, T.untyped]),
+          mapper_callable: T.nilable(T.proc.params(arg0: T.untyped).returns(T.untyped)),
+        ).void
+      end
+      def initialize(index_base_name:, index_settings:, index_mappings:, mapper_callable:); end
+
+      sig { returns(String) }
+      def archived_alias
+        "#{base_alias_name}.archive"
+      end
+
+      sig { returns(String) }
+      def active_alias
+        base_alias_name
+      end
+
+      sig { returns(String) }
+      def new_alias
+        "#{base_alias_name}.new"
+      end
+
+      sig { abstract.returns(String) }
+      def read_alias_name; end
+
+      sig { abstract.params(record: PostAddressData).returns(T::Hash[Symbol, T.untyped]) }
+      def record_source(record); end
+
+      sig { abstract.returns(String) }
+      def base_alias_name; end
+
+      sig do
+        abstract.params(
+          ensure_clean: T::Boolean,
+          raise_errors: T::Boolean,
+        ).void
+      end
+      def create_next_index(ensure_clean: false, raise_errors: false); end
+
+      sig do
+        abstract.params(
+          raise_errors: T::Boolean,
+        ).void
+      end
+      def switch_to_next_index(raise_errors: false); end
+
+      sig do
+        abstract.params(
+          records: T.any(ActiveRecord::Relation, T::Array[PostAddressData]),
+        ).returns(T.nilable(Response))
+      end
+      def save_records_backfill(records); end
+
+      sig { abstract.params(query: T::Hash[String, T.untyped]).returns(T::Hash[String, T.untyped]) }
+      def search(query); end
+
+      sig { abstract.params(id: T.any(String, Integer)).returns(T::Hash[String, T.untyped]) }
+      def find(id); end
+
+      sig { abstract.params(query: T::Hash[String, T.untyped]).returns(T::Hash[String, T.untyped]) }
+      def analyze(query); end
+
+      sig { abstract.params(query: T::Hash[String, T.untyped]).returns(T::Hash[String, T.untyped]) }
+      def term_vectors(query); end
+    end
+  end
+end

data/app/models/atlas_engine/elasticsearch/response.rb

@@ -0,0 +1,20 @@
+# typed: strict
+# frozen_string_literal: true
+
+module AtlasEngine
+  module Elasticsearch
+    class Response
+      extend T::Sig
+
+      sig { returns(T.untyped) }
+      attr_reader :response
+
+      delegate :body, :status, :headers, to: :response
+
+      sig { params(response: T.untyped).void }
+      def initialize(response)
+        @response = response
+      end
+    end
+  end
+end

data/app/models/atlas_engine/event.rb

@@ -0,0 +1,12 @@
+# typed: true
+# frozen_string_literal: true
+
+module AtlasEngine
+  class Event < ApplicationRecord
+    include ActiveModel::Validations
+
+    belongs_to :country_import, class_name: "AtlasEngine::CountryImport"
+    validates :message, presence: true
+    enum category: { progress: 0, error: 1, invalid_address: 2 }
+  end
+end

data/app/models/atlas_engine/field_decompounder.rb

@@ -0,0 +1,36 @@
+# typed: true
+# frozen_string_literal: true
+
+module AtlasEngine
+  class FieldDecompounder
+    extend T::Sig
+
+    attr_reader :field, :value, :country_profile
+
+    sig { params(field: Symbol, value: T.nilable(String), country_profile: CountryProfile).void }
+    def initialize(field:, value:, country_profile:)
+      @field = field
+      @value = value
+      @country_profile = country_profile
+    end
+
+    sig { returns(T.nilable(String)) }
+    def call
+      country_profile.decompounding_patterns(field)&.each do |pattern|
+        transliterated_street = ActiveSupport::Inflector.transliterate(value.to_s)
+        if (match = transliterated_street.match(expanded_pattern(pattern)))
+          return (match[:pre] + match[:name] + " " + match[:suffix] + match[:post]).strip
+        end
+      end
+
+      value
+    end
+
+    private
+
+    sig { params(pattern: String).returns(Regexp) }
+    def expanded_pattern(pattern)
+      /(?<pre>.*\b)#{pattern}(?<post>.*)/i
+    end
+  end
+end

data/app/models/atlas_engine/index_configuration_factory.rb

@@ -0,0 +1,188 @@
+# typed: strict
+# frozen_string_literal: true
+
+module AtlasEngine
+  class IndexConfigurationFactory
+    extend T::Sig
+
+    IndexConfigurations = T.type_alias { T::Hash[T.any(String, Symbol), T.untyped] }
+
+    INDEX_CONFIGURATIONS_ROOT = T.let(
+      File.join(AtlasEngine::Engine.root, "db/data/address_synonyms/index_configurations"), String
+    )
+    COUNTRIES_ROOT = T.let(File.join(AtlasEngine::Engine.root, "app/countries/atlas_engine"), String)
+
+    DEFAULT_NUMBER_OF_SHARDS = 1
+    DEFAULT_NUMBER_OF_REPLICAS = 1
+    DEFAULT_MIN_ZIP_EDGE_GRAM = 1
+    DEFAULT_MAX_ZIP_EDGE_GRAM = 10
+
+    sig { returns(Symbol) }
+    attr_reader :country_code
+
+    sig { returns(T.nilable(String)) }
+    attr_reader :locale
+
+    sig { returns(CountryProfile) }
+    attr_reader :country_profile
+
+    sig { returns(T.nilable(Integer)) }
+    attr_reader :shard_override
+
+    sig { returns(T.nilable(Integer)) }
+    attr_reader :replica_override
+
+    sig do
+      params(
+        country_code: String,
+        locale: T.nilable(String),
+        shard_override: T.nilable(Integer),
+        replica_override: T.nilable(Integer),
+      ).void
+    end
+    def initialize(
+      country_code:,
+      locale: nil,
+      shard_override: nil,
+      replica_override: nil
+    )
+      @country_code = T.let(country_code.downcase.to_sym, Symbol)
+      @locale = T.let(locale.present? ? locale.downcase : nil, T.nilable(String))
+      @country = T.let(Worldwide.region(code: @country_code), Worldwide::Region)
+      @country_profile = T.let(CountryProfile.for(@country_code.to_s.upcase, @locale), CountryProfile)
+      @shard_override = T.let(shard_override, T.nilable(Integer))
+      @replica_override = T.let(replica_override, T.nilable(Integer))
+    end
+
+    sig do
+      params(
+        creating: T::Boolean,
+      ).returns(IndexConfigurations)
+    end
+    def index_configuration(creating: false)
+      remove_create_only_settings unless creating
+      configuration
+    end
+
+    private
+
+    sig { returns(IndexConfigurations) }
+    def configuration
+      @configuration ||= T.let(
+        defaults.deep_merge(country_overrides).deep_merge(locale_overrides),
+        T.nilable(IndexConfigurations),
+      )
+    end
+
+    sig { returns(IndexConfigurations) }
+    def defaults
+      render_file(:default)
+    end
+
+    sig { returns(IndexConfigurations) }
+    def country_overrides
+      render_file(country_code)
+    end
+
+    sig { returns(IndexConfigurations) }
+    def locale_overrides
+      return {} if locale.blank?
+
+      render_file(country_code, locale)
+    end
+
+    sig { params(country_code: Symbol, locale: T.nilable(String)).returns(IndexConfigurations) }
+    def render_file(country_code, locale = nil)
+      file_path = file_path(country_code, locale)
+
+      return {} if file_path.nil?
+
+      configuration_file = ActiveSupport::ConfigurationFile.new(file_path)
+      configuration_file.parse(context: binding).deep_stringify_keys
+    end
+
+    sig { params(country_code: Symbol, locale: T.nilable(String)).returns(T.nilable(String)) }
+    def file_path(country_code, locale = nil)
+      if country_code == :default
+        return File.join(INDEX_CONFIGURATIONS_ROOT, "default.yml")
+      end
+
+      path = if locale.present?
+        File.join(COUNTRIES_ROOT, "#{country_code}/locales/#{locale}/index_configuration.yml")
+      else
+        File.join(COUNTRIES_ROOT, "#{country_code}/index_configuration.yml")
+      end
+
+      path if File.file?(path)
+    end
+
+    sig { void }
+    def remove_create_only_settings
+      configuration.dig("settings", "index")&.delete("number_of_shards")
+      configuration.dig("settings", "index")&.delete("number_of_replicas")
+    end
+
+    sig { returns(T::Array[String]) }
+    def city_synonyms
+      synonyms[:city_synonyms] || []
+    end
+
+    sig { returns(T::Array[String]) }
+    def street_synonyms
+      synonyms[:street_synonyms] || []
+    end
+
+    sig { returns(T::Hash[Symbol, T::Array[String]]) }
+    def synonyms
+      @synonyms ||= T.let(
+        begin
+          file_name = if locale.present?
+            "#{COUNTRIES_ROOT}/#{country_code}/locales/#{locale}/synonyms.yml"
+          else
+            "#{COUNTRIES_ROOT}/#{country_code}/synonyms.yml"
+          end
+
+          empty_synonyms_hash = { street_synonyms: [], city_synonyms: [] }
+
+          File.file?(file_name) ? YAML.load_file(file_name, freeze: true).deep_symbolize_keys : empty_synonyms_hash
+        end,
+        T.nilable(T::Hash[Symbol, T::Array[String]]),
+      )
+    end
+
+    sig { returns(String) }
+    def number_of_shards
+      (
+        shard_override ||
+        country_profile.ingestion.settings_number_of_shards ||
+        DEFAULT_NUMBER_OF_SHARDS
+      ).to_s
+    end
+
+    sig { returns(String) }
+    def number_of_replicas
+      (
+        replica_override ||
+        country_profile.ingestion.settings_number_of_replicas ||
+        DEFAULT_NUMBER_OF_REPLICAS
+      ).to_s
+    end
+
+    sig { returns(String) }
+    def zip_edge_min_gram
+      country_profile.ingestion.settings_min_zip_edge_ngram || DEFAULT_MIN_ZIP_EDGE_GRAM.to_s
+    end
+
+    sig { returns(T.nilable(String)) }
+    def zip_edge_max_gram
+      country_profile.ingestion.settings_max_zip_edge_ngram ||
+        @country.zip_example&.length&.to_s ||
+        DEFAULT_MAX_ZIP_EDGE_GRAM.to_s
+    end
+
+    sig { returns(T::Boolean) }
+    def country_has_zip?
+      @country.has_zip?
+    end
+  end
+end

data/app/models/atlas_engine/post_address.rb

@@ -0,0 +1,114 @@
+# typed: false
+# frozen_string_literal: true
+
+module AtlasEngine
+  class PostAddress < ApplicationRecord
+    extend T::Sig
+
+    validates :country_code, :city, presence: true
+    validate :country_must_be_supported, :province_must_match_country, :zip_must_match_country_and_province
+    serialize :city, type: Array, coder: YAML
+
+    sig { returns(T::Hash[Symbol, T.untyped]) }
+    def to_h
+      {
+        source_id: source_id,
+        locale: locale,
+        country_code: country_code,
+        province_code: province_code,
+        region1: region1,
+        region2: region2,
+        region3: region3,
+        region4: region4,
+        city: city,
+        suburb: suburb,
+        zip: zip,
+        street: street,
+        building_name: building_name,
+        latitude: latitude,
+        longitude: longitude,
+      }.compact
+    end
+
+    private
+
+    sig { void }
+    def country_must_be_supported
+      if country_code.blank?
+        errors.add(:country, "is required")
+      elsif !country.country?
+        errors.add(:country, "with code '#{country_code}' is not recognized") unless country.country?
+      end
+    end
+
+    sig { void }
+    def province_must_match_country
+      return if country_code.blank?
+      return unless country_has_provinces?
+
+      if province_missing?
+        errors.add(:province_code, "is required for country '#{country_code}'") if province_missing?
+      elsif province_invalid?
+        errors.add(:province_code, "'#{province_code}' is invalid for country '#{country_code}'")
+      end
+    end
+
+    sig { void }
+    def zip_must_match_country_and_province
+      return if country_code.blank?
+
+      zip_must_match_country
+      return if errors[:zip].any?
+
+      zip_must_match_province
+    end
+
+    sig { void }
+    def zip_must_match_country
+      return unless country_has_zip_codes?
+
+      if zip.blank? && country.zip_required?
+        errors.add(:zip, "is required for country '#{country_code}'")
+      elsif !country.valid_zip?(zip)
+        errors.add(:zip, "'#{zip}' is invalid for country '#{country_code}'")
+      end
+    end
+
+    sig { void }
+    def zip_must_match_province
+      return unless province_code
+
+      province = country.zone(code: province_code)
+
+      return unless province.province?
+      return if province.valid_zip?(zip)
+
+      errors.add(:zip, "'#{zip}' is invalid for province '#{province_code}'")
+    end
+
+    sig { returns(Worldwide::Region) }
+    def country
+      @country ||= T.let(Worldwide.region(code: country_code), T.nilable(Worldwide::Region))
+    end
+
+    sig { returns(T::Boolean) }
+    def country_has_provinces?
+      country.zones.present? && !country.hide_provinces_from_addresses
+    end
+
+    sig { returns(T::Boolean) }
+    def country_has_zip_codes?
+      country.country? && country.has_zip?
+    end
+
+    sig { returns(T::Boolean) }
+    def province_invalid?
+      province_code.present? && !country.zone(code: province_code).province?
+    end
+
+    sig { returns(T::Boolean) }
+    def province_missing?
+      province_code.blank? && !country.province_optional?
+    end
+  end
+end