atlas_engine 0.1.0

data/app/models/atlas_engine/post_address_importer.rb

@@ -0,0 +1,34 @@
+# typed: false
+# frozen_string_literal: true
+
+require "csv"
+
+module AtlasEngine
+  class PostAddressImporter
+    def initialize(file)
+      @file = file
+    end
+
+    def import
+      CSV.foreach(@file, headers: true) do |row|
+        PostAddress.create!(
+          source_id: row["source_id"],
+          locale: row["locale"],
+          country_code: row["country_code"],
+          province_code: row["province_code"],
+          region1: row["region1"],
+          region2: row["region2"],
+          region3: row["region3"],
+          region4: row["region4"],
+          city: row["city"],
+          suburb: row["suburb"],
+          zip: row["zip"],
+          street: row["street"],
+          building_name: row["building_name"],
+          latitude: row["latitude"],
+          longitude: row["longitude"],
+        )
+      end
+    end
+  end
+end

data/app/models/atlas_engine/services/service_helper.rb

@@ -0,0 +1,21 @@
+# typed: true
+# frozen_string_literal: true
+
+module AtlasEngine
+  module Services
+    module ServiceHelper
+      def handle_metrics(request_type, country_code, using_session_token)
+        result = yield
+
+        ActiveSupport::Notifications.instrument("atlas.service.events", {
+          request_type: request_type,
+          country_code: country_code,
+          using_session_token: using_session_token,
+          result: result,
+        })
+
+        result
+      end
+    end
+  end
+end

data/app/models/atlas_engine/services/validation.rb

@@ -0,0 +1,65 @@
+# typed: true
+# frozen_string_literal: true
+
+module AtlasEngine
+  module Services
+    class Validation
+      extend T::Sig
+      VALIDATE_ADDRESS = "validate_address"
+
+      class << self
+        extend T::Sig
+        include ServiceHelper
+        T.unsafe(self).include(AtlasEngine.validation_eligibility.constantize)
+
+        sig { params(request: AddressValidation::Request).returns(AddressValidation::Result) }
+        def validate_address(request)
+          country_code = request.address.country_code
+          handle_metrics(Validation::VALIDATE_ADDRESS, country_code, false) do
+            matching_strategy = if validation_enabled?(request.address)
+              serialize_strategy(request.matching_strategy, request.address)
+            else
+              AddressValidation::MatchingStrategies::Local
+            end
+
+            validator = AddressValidation::Validator.new(
+              address: request.address,
+              locale: request.locale,
+              matching_strategy: matching_strategy,
+              context: request.address.context.to_h,
+            )
+            result = validator.run
+
+            AddressValidation::StatsdEmitter.new(address: request.address, result: result).run
+            AddressValidation::LogEmitter.new(address: request.address, result: result).run
+
+            if Rails.configuration.x.captured_concerns.enabled
+              AddressValidation::ConcernProducer.add(result, request.address.context.to_h)
+            end
+
+            result
+          end
+        end
+
+        private
+
+        sig do
+          params(
+            request_matching_strategy: T.nilable(T.any(String, Symbol)),
+            address: AddressValidation::AbstractAddress,
+          ).returns(AddressValidation::MatchingStrategies)
+        end
+        def serialize_strategy(request_matching_strategy, address)
+          requested_strategy = if request_matching_strategy.nil?
+            CountryProfile.for(T.must(address.country_code)).validation.default_matching_strategy
+          else
+            request_matching_strategy.to_s.downcase
+          end
+
+          serialized_strategy = AddressValidation::MatchingStrategies.try_deserialize(requested_strategy)
+          serialized_strategy.presence || AddressValidation::MatchingStrategies::Local
+        end
+      end
+    end
+  end
+end

data/app/models/atlas_engine/services/validation_eligibility.rb

@@ -0,0 +1,18 @@
+# typed: true
+# frozen_string_literal: true
+
+module AtlasEngine
+  module Services
+    module ValidationEligibility
+      extend T::Sig
+
+      sig { params(address: AddressValidation::AbstractAddress).returns(T::Boolean) }
+      def validation_enabled?(address)
+        return false if address.country_code.blank?
+        return true unless Rails.env.production? || Rails.env.test?
+
+        CountryProfile.for(T.must(address.country_code)).validation.enabled
+      end
+    end
+  end
+end

data/app/models/atlas_engine/street.rb

@@ -0,0 +1,34 @@
+# typed: true
+# frozen_string_literal: true
+
+module AtlasEngine
+  class Street
+    extend T::Sig
+
+    attr_reader :street
+
+    sig { params(street: String).void }
+    def initialize(street:)
+      @street = street
+    end
+
+    sig { returns(T.nilable(String)) }
+    def name
+      parsing[:name]
+    end
+
+    sig { returns(String) }
+    def with_stripped_name
+      return street if name.blank?
+
+      street.sub(name, T.must(name).gsub(/\s+/, ""))
+    end
+
+    private
+
+    sig { returns(T::Hash[Symbol, String]) }
+    def parsing
+      @parsing ||= AtlasEngine::ValidationTranscriber::StreetParser.new.parse(street: street)
+    end
+  end
+end

data/app/tasks/maintenance/atlas_engine/elasticsearch_index_create_task.rb

@@ -0,0 +1,106 @@
+# typed: true
+# frozen_string_literal: true
+
+module Maintenance
+  module AtlasEngine
+    class ElasticsearchIndexCreateTask < MaintenanceTasks::Task
+      include ::AtlasEngine::LogHelper
+      extend T::Sig
+
+      attribute :country_code, :string
+      attribute :locale, :string # optional (no locale implies index = country_code)
+      attribute :province_codes, :string # optional comma separated list of province codes
+      attribute :shard_override, :integer, default: nil
+      attribute :replica_override, :integer, default: nil
+      attribute :activate_index, :boolean, default: false unless Rails.env.production?
+
+      validates :country_code, presence: true
+
+      attr_writer :repository
+
+      after_complete :switch_index unless Rails.env.production?
+
+      sig { returns(T.nilable(ActiveRecord::Batches::BatchEnumerator)) }
+      def collection
+        @batch_number = 0
+        batch_size = 2000
+
+        sanitized_country_code = validate(T.must(country_code))
+
+        address_conditions = {
+          country_code: sanitized_country_code,
+          province_code: sanitized_province_codes,
+          locale: sanitized_locale,
+        }.compact_blank
+
+        record_count = ::AtlasEngine::PostAddress.where(address_conditions).size
+        raise "No records to process for country code: #{country_code}" if record_count.zero?
+
+        repository.create_next_index(ensure_clean: true)
+
+        ::AtlasEngine::PostAddress.where(address_conditions).in_batches(of: batch_size)
+      end
+
+      sig { params(batch_of_post_address: ActiveRecord::Relation).void }
+      def process(batch_of_post_address)
+        log_info("Processing batch #{@batch_number} for repository #{repository.read_alias_name.upcase}.")
+        repository.save_records_backfill(batch_of_post_address)
+        @batch_number += 1
+      end
+
+      sig { returns(::AtlasEngine::CountryRepository) }
+      def repository
+        @repository ||= ::AtlasEngine::CountryRepository.new(
+          country_code: T.must(country_code),
+          repository_class: ::AtlasEngine.elasticsearch_repository.constantize,
+          locale: sanitized_locale,
+          index_configuration: index_configuration,
+        )
+      end
+
+      sig { void }
+      def switch_index
+        if activate_index
+          repository.switch_to_next_index
+          log_info("Switched index `#{repository.read_alias_name}` live.")
+        end
+      end
+
+      private
+
+      sig { params(country_code: String).returns(String) }
+      def validate(country_code)
+        region = Worldwide.region(code: country_code)
+        unless region.country?
+          raise ArgumentError, "Invalid country code: #{country_code}"
+        end
+
+        region.iso_code
+      end
+
+      sig { returns(T::Array[String]) }
+      def sanitized_province_codes
+        return [] if province_codes.blank?
+
+        T.must(province_codes).downcase.split(",").map(&:strip)
+      end
+
+      sig { returns(String) }
+      def sanitized_locale
+        locale.to_s.downcase
+      end
+
+      sig { returns(::AtlasEngine::IndexConfigurationFactory::IndexConfigurations) }
+      def index_configuration
+        ::AtlasEngine::IndexConfigurationFactory.new(
+          country_code: T.must(country_code),
+          locale: sanitized_locale,
+          shard_override: shard_override,
+          replica_override: replica_override,
+        ).index_configuration(
+          creating: true,
+        )
+      end
+    end
+  end
+end

data/app/tasks/maintenance/atlas_engine/geo_json_import_task.rb

@@ -0,0 +1,29 @@
+# typed: false
+# frozen_string_literal: true
+
+module Maintenance
+  module AtlasEngine
+    class GeoJsonImportTask < MaintenanceTasks::Task
+      include ::AtlasEngine::HandlesBlob
+
+      no_collection
+
+      attribute :clear_records, :boolean, default: true
+      # ISO3166 two-letter country code.
+      attribute :country_code, :string
+      validates :country_code, presence: true
+      # Filename to import. When running in staging or production, the worker expects to find
+      # this file in the relevant GCS bucket, configured in `config/storage/{environment}.yml`
+      # It must be placed under `openaddress/` with the same filename.
+      attribute :geojson_file_path, :string
+      attribute :locale, :string
+
+      def process = ::AtlasEngine::AddressImporter::OpenAddress::GeoJsonImportLauncherJob.perform_later(
+        country_code:,
+        geojson_file_path: geojson_file_path.strip,
+        clear_records:,
+        locale:,
+      )
+    end
+  end
+end

data/app/views/atlas_engine/connectivity/index.html.erb

@@ -0,0 +1,50 @@
+<h2>Elasticsearch Indices</h2>
+<div id="elasticsearch_indices">
+  <table class="table table-striped table-bordered">
+    <thead>
+      <tr>
+        <th>Name</th>
+        <th>Size</th>
+        <th>Document Count</th>
+      </tr>
+    </thead>
+
+    <tbody>
+      <% @indices&.each do |index| %>
+        <tr>
+          <td><%= index["index"] %></td>
+          <td><%= index["store.size"] %></td>
+          <td><%= index["docs.count"] %></td>
+        </tr>
+      <% end %>
+    </tbody>
+  </table>
+</div>
+
+<h2>Post Address Records</h2>
+<div id="post_addresses">
+  <table class="table table-striped table-bordered">
+    <thead>
+      <tr>
+        <th>ID</th>
+        <th>Street</th>
+        <th>City</th>
+        <th>Zip</th>
+        <th>Province Code</th>
+      </tr>
+    </thead>
+
+    <tbody>
+      <% @post_addresses&.each do |address| %>
+        <tr>
+          <td><%= address.id %></td>
+          <td><%= address.street %></td>
+          <td><%= address.city %></td>
+          <td><%= address.zip %></td>
+          <td><%= address.province_code %></td>
+        </tr>
+      <% end %>
+    </tbody>
+  </table>
+</div>
+

data/app/views/atlas_engine/country_imports/index.html.erb

@@ -0,0 +1,49 @@
+<h1>Country Imports</h1>
+
+<div id="country_imports">
+<table class="table table-striped table-bordered">
+  <thead>
+    <tr>
+      <th>ID</th>
+      <th>country</th>
+      <th>state</th>
+      <th>created_at</th>
+      <th>updated_at</th>
+      <th>actions</th>
+    </tr>
+  </thead>
+
+  <tbody>
+    <% @country_imports.each do |import| %>
+      <tr>
+        <td><%=link_to import.id, country_import_path(import.id) %></td>
+        <td><%= import.country_code %></td>
+        <td><%= import.state %></td>
+        <td><%= import.created_at%></td>
+        <td><%= import.updated_at %></td>
+        <% if import.in_progress? || import.pending? %>
+          <td><%= link_to "Interrupt", interrupt_country_import_path(import.id), class: "btn btn-danger" %></td>
+        <% else %>
+          <td><p>-</p></td>
+        <% end %>
+      </tr>
+    <% end %>
+  </tbody>
+</table>
+</div>
+
+<div class="container">
+  <div class="span2">
+    <% if @page > 0 %>
+      <%= link_to "← Previous", country_imports_path(page: @page - 1), type: "button", class: "btn btn-primary btn-block" %>
+    <% else %>
+      <%= link_to "← Previous", country_imports_path(page: @page - 1), type: "button", class: "btn btn-primary btn-block disabled" %>
+    <% end %>
+
+    <% if AtlasEngine::CountryImport.count > ((@page+1)*AtlasEngine::CountryImport::PAGE_SIZE) %>
+       <%= link_to "Next →", country_imports_path(page: @page + 1),type: "button",  class:"btn btn-primary btn-block"%>
+    <% else %>
+       <%= link_to "Next →", country_imports_path(page: @page + 1),type: "button",  class:"btn btn-primary btn-block disabled"%>
+    <% end %>
+  </div>
+</div>

data/app/views/atlas_engine/country_imports/show.html.erb

@@ -0,0 +1,73 @@
+<head>
+  <meta http-equiv="refresh" content="15">
+</head>
+<h1><%= "#{t('admin.events.country_import.title')} #{@country_import&.id}" %></h1>
+<br>
+
+<% if @country_import&.in_progress? || @country_import&.pending? %>
+  <%= link_to "Interrupt", interrupt_country_import_path(@country_import.id), class: "btn btn-danger" %>
+  <br>
+  <br>
+<% end %>
+
+<div id="country_imports">
+  <table class="table table-striped table-bordered">
+    <tr>
+      <th><%= t('admin.events.country_import.country') %></th>
+      <td>
+        <%=@country_import&.country_code %>
+      </td>
+    </tr>
+    <tr>
+      <th><%= t('admin.events.country_import.state') %></th>
+      <td>
+        <%=@country_import&.state %>
+      </td>
+    </tr>
+    <tr>
+      <th><%= t('admin.events.country_import.created_at') %></th>
+      <td>
+        <% unless @country_import.nil? %>
+          <%="#{time_ago_in_words(@country_import&.created_at)} ago (#{@country_import&.created_at})"%>
+        <% end %>
+      </td>
+    </tr>
+    <tr>
+      <th><%= t('admin.events.country_import.updated_at') %></th>
+      <td>
+        <% unless @country_import.nil? %>
+          <%="#{time_ago_in_words(@country_import&.updated_at)} ago (#{@country_import&.updated_at})"%>
+        <% end %>
+      </td>
+    </tr>
+  </tbody>
+</table>
+<h1><%= t('admin.events.events.title') %></h1>
+<br>
+<div id="country_imports">
+  <table class="table table-striped table-bordered">
+    <thead>
+      <tr>
+        <th><%= t('admin.events.events.created_at') %></th>
+        <th><%= t('admin.events.events.message') %></th>
+        <th><%= t('admin.events.events.additional_params') %></th>
+      </tr>
+    </thead>
+    <tbody>
+      <% @events.each do |event| %>
+        <tr>
+          <td>
+            <%="#{time_ago_in_words(event.created_at)} ago (#{event.created_at})"%>
+          </td>
+          <td>
+            <%=event.message %>
+          </td>
+            <% if event.additional_params %>
+          <td><%= event.additional_params  %></td>
+        <% else %>
+          <td><p>-</p></td>
+        <% end %>
+        </tr>
+      <% end %>
+    </tbody>
+  </table>

data/app/views/layouts/atlas_engine/application.html.erb

@@ -0,0 +1,15 @@
+<!DOCTYPE html>
+<html>
+<head>
+  <title>Atlas engine</title>
+  <%= csrf_meta_tags %>
+  <%= csp_meta_tag %>
+
+  <%= stylesheet_link_tag    "atlas_engine/application", media: "all" %>
+</head>
+<body>
+
+<%= yield %>
+
+</body>
+</html>

data/config/initializers/1.ruby_patches.rb

@@ -0,0 +1,18 @@
+# typed: true
+# frozen_string_literal: true
+
+module Enumerable
+  extend T::Sig
+
+  raise "Enumerable#stable_sort_by is already defined" if method_defined?(:stable_sort_by)
+
+  sig do
+    params(
+      block: T.proc.params(arg0: T.untyped).returns(T.any(Comparable, T::Array[BasicObject])),
+    )
+      .returns(T::Array[T.untyped])
+  end
+  def stable_sort_by(&block)
+    sort_by.with_index { |x, i| [yield(x), i] }
+  end
+end

data/config/initializers/sorbet.rb

@@ -0,0 +1,5 @@
+# typed: true
+# frozen_string_literal: true
+
+# Includes `extend T::Sig` in every class
+Module.include(T::Sig)

data/config/initializers/worldwide.rb

@@ -0,0 +1,5 @@
+# typed: false
+# frozen_string_literal: true
+
+I18n.available_locales = Worldwide::Locales.known
+Worldwide::Config.configure_i18n

data/config/locales/internal/en.yml

@@ -0,0 +1,14 @@
+en:
+  admin:
+    events:
+      country_import:
+        title: "Country Import"
+        country: "country:"
+        state: "state:"
+        created_at: "created_at:"
+        updated_at: "updated_at:"
+      events:
+        title: "Events"
+        created_at: "created_at"
+        message: "message"
+        additional_params: "additional params"

data/config/routes.rb

@@ -0,0 +1,17 @@
+# typed: false
+# frozen_string_literal: true
+
+AtlasEngine::Engine.routes.draw do
+  mount MaintenanceTasks::Engine => "/maintenance_tasks"
+  if Rails.env.local?
+    mount GraphiQL::Rails::Engine, at: "/graphiql", graphql_path: "/graphql"
+  end
+  post "/graphql", to: "graphql#execute"
+  get "/connectivity", to: "connectivity#index"
+
+  resources :country_imports do
+    member do
+      get "interrupt"
+    end
+  end
+end