atlas_engine 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 3d590dd6d627aea5298889faf7dccb0ec3479277beb107750a9da288c6c8aefa
+  data.tar.gz: 349e59b38ed5e20ee84c0b27cd660b619e789ff5755aa80af26a4e04e55bce5f
+SHA512:
+  metadata.gz: b0bea20736bd75159f04ad10c7f2f0b0805b0d5425b7c43c7d2426bd249a61ed28b80ca9ca20bae0190b91b5999771e2a6c9cf2780a2dc9e8693bb0073118dd7
+  data.tar.gz: 32c0509404553da96ef197a6ff01ae5e82a433e09d8a36ee27a0ae1b1fe1abce1be3b07c17d85ae7b52ba3eaee76626252e6298c33cc7d9036050ede3866acd5

data/README.md

@@ -0,0 +1,123 @@
+# Atlas Engine
+
+This is a rails engine that is meant to provide end-to-end address validation for rails apps
+
+## Local Setup
+
+### In your rails app
+
+#### Initial setup
+Add the engine to your gemfile
+```
+gem "atlas_engine", git: "https://github.com/Shopify/atlas-engine"
+```
+
+Run the following commands to install the engine in your rails app
+
+```
+bundle lock
+bin/rails generate atlas_engine:install
+```
+
+#### Updating to a newer version of the engine
+
+Working with migrations
+```
+# Copy any migrations from the engine into your app
+rails atlas_engine:install:migrations
+
+# Perform the migrations in your app
+rails db:migrate
+```
+
+### Developing in the engine
+
+#### Setup Docker
+
+```
+brew install docker
+brew install docker-compose
+
+# to setup the docker daemon
+brew install colima
+
+# to start the docker daemon
+colima start --cpu 4 --memory 8
+colima ssh
+  sudo sysctl -w vm.max_map_count=262144
+  exit
+
+```
+
+Verify if docker is running: `docker info`
+
+#### Setup Rails
+
+```
+bundle install
+
+# If you get an ssl error with puma installation run
+bundle config build.puma --with-pkg-config=$(brew --prefix openssl@3)/lib/pkgconfig
+```
+
+#### Setting up Elasticsearch, Mysql
+
+```
+bash setup
+docker-compose up
+
+# If you encounter an error getting docker credentials, remove or update the `credsStore`
+key in your Docker configuration file:
+
+# ~/.docker/config.json
+"credsStore": "desktop", # remove this line
+```
+
+Connecting to Docker services
+  * for Mysql : `mysql --host=127.0.0.1 --user=user --password=changeme`
+  * for Elasticsearch : `http://localhost:9200`
+
+  _note: if you have updated any of the ports in your .env file then use those ports instead_
+
+
+#### Setting up db
+```
+rails db:setup
+```
+
+#### Setting up maintenance tasks
+After locally setting up Atlas Engine:
+```
+rails app:maintenance_tasks:install:migrations
+rails db:migrate
+```
+
+## Using the App
+
+### Infrastructure Requirements
+The elasticsearch implementation depends on the ICU analysis plugin. Refer to the [Dockfile](./Dockfile) leveraged in local setup for plugin installation.
+
+### Starting the App and Testing
+  * `bin/rails server` to start the server
+  * `bin/rails test` to run tests
+
+### Running Sorbet
+
+Generate rbi files for custom code
+```
+bin/tapioca dsl --app-root="test/dummy/"
+```
+
+Generate rbi files for gems
+```
+bin/tapioca gems
+
+# or
+
+bin/tapioca gems --all
+```
+
+Running a sorbet check
+```
+srb tc
+```

data/Rakefile

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+# # frozen_string_literal: true
+
+# # Add your own tasks in files placed in lib/tasks ending in .rake,
+# # for example lib/tasks/capistrano.rake, and they will automatically be available to Rake.
+
+# require_relative "config/application"
+
+# Rails.application.load_tasks
+
+# from generate plugin --mountable
+require "bundler/setup"
+
+APP_RAKEFILE = File.expand_path("test/dummy/Rakefile", __dir__)
+load "rails/tasks/engine.rake"
+
+load "rails/tasks/statistics.rake"
+
+require "bundler/gem_tasks"

data/app/assets/config/atlas_engine_manifest.js

@@ -0,0 +1,3 @@
+//= link_directory ../stylesheets/atlas_engine .css
+//= link graphiql/rails/application.css
+//= link graphiql/rails/application.js

data/app/assets/stylesheets/atlas_engine/application.css

@@ -0,0 +1,15 @@
+/*
+ * This is a manifest file that'll be compiled into application.css, which will include all the files
+ * listed below.
+ *
+ * Any CSS and SCSS file within this directory, lib/assets/stylesheets, vendor/assets/stylesheets,
+ * or any plugin's vendor/assets/stylesheets directory can be referenced here using a relative path.
+ *
+ * You're free to add application-wide styles to this file and they'll appear at the bottom of the
+ * compiled file so the styles you add here take precedence over styles defined in any other CSS/SCSS
+ * files in this directory. Styles in this file should be added after the last require_* statement.
+ * It is generally better to create a new file per style scope.
+ *
+ *= require_tree .
+ *= require_self
+ */

data/app/concerns/atlas_engine/handles_blob.rb

@@ -0,0 +1,26 @@
+# typed: false
+# frozen_string_literal: true
+
+module AtlasEngine
+  module HandlesBlob
+    extend ActiveSupport::Concern
+
+    class BlobNotFoundError < StandardError; end
+
+    included do
+      def storage_service
+        ActiveStorage::Blob.service
+      end
+
+      def blob_exists?(key)
+        storage_service.exist?(key)
+      end
+
+      def download(key, &block)
+        raise BlobNotFoundError unless blob_exists?(key)
+
+        storage_service.open(key, verify: false, &block)
+      end
+    end
+  end
+end

data/app/concerns/atlas_engine/handles_interruption.rb

@@ -0,0 +1,22 @@
+# typed: true
+# frozen_string_literal: true
+
+module AtlasEngine
+  module HandlesInterruption
+    extend ActiveSupport::Concern
+    include LogHelper
+    include Kernel
+
+    included do
+      def exit_if_interrupted!(import)
+        import_state = Rails.cache.fetch("country_import:#{import.id}:state", expires_in: 5.seconds) do
+          import.reload.state
+        end
+
+        return unless import_state == "failed"
+
+        raise CountryImport::InterruptionError, "Import interrupted at #{Time.current.utc.to_fs}"
+      end
+    end
+  end
+end

data/app/controllers/atlas_engine/application_controller.rb

@@ -0,0 +1,7 @@
+# typed: false
+# frozen_string_literal: true
+
+module AtlasEngine
+  class ApplicationController < ActionController::Base
+  end
+end

data/app/controllers/atlas_engine/connectivity_controller.rb

@@ -0,0 +1,21 @@
+# typed: false
+# frozen_string_literal: true
+
+module AtlasEngine
+  class ConnectivityController < ApplicationController
+    def initialize
+      super
+      @repository = AtlasEngine::Elasticsearch::Repository.new(
+        index_base_name: "",
+        index_settings: {},
+        index_mappings: {},
+        mapper_callable: nil,
+      )
+    end
+
+    def index
+      @indices = @repository.indices
+      @post_addresses = AtlasEngine::PostAddress.limit(10)
+    end
+  end
+end

data/app/controllers/atlas_engine/country_imports_controller.rb

@@ -0,0 +1,73 @@
+# typed: strict
+# frozen_string_literal: true
+
+module AtlasEngine
+  class CountryImportsController < ApplicationController
+    extend T::Sig
+
+    before_action :set_page, only: [:index]
+    before_action :set_country_import, only: [:interrupt, :show]
+
+    include LogHelper
+    layout "application"
+
+    sig { void }
+    def initialize
+      super
+      @page = T.let(0, Integer)
+      @country_import = T.let(T.unsafe(nil), T.nilable(CountryImport), checked: false)
+      @country_imports = T.let(T.unsafe(nil), ActiveRecord::Relation, checked: false)
+      @events = T.let(T.unsafe(nil), ActiveRecord::Relation, checked: false)
+    end
+
+    # GET /country_imports
+    sig { void }
+    def index
+      @country_imports = CountryImport.order(id: :desc)
+        .offset(CountryImport::PAGE_SIZE * @page)
+        .limit(CountryImport::PAGE_SIZE)
+    end
+
+    sig { void }
+    def show
+      @events = Event.where(country_import: @country_import).order(id: :desc).limit(1000)
+    end
+
+    # GET /country_imports/1/interrupt
+    sig { void }
+    def interrupt
+      T.must(@country_import).interrupt!
+
+      log_info("Country import manually interrupted", { country_import_id: T.must(@country_import).id })
+      redirect_back_or_to(
+        country_imports_url,
+        notice: "Country import manually interrupted. Wait a few minutes for ongoing jobs to stop.",
+      )
+    rescue => e
+      log_error(
+        "Country import manual interruption failed with #{e.class} - #{e.message}; stack trace #{e.backtrace.inspect}",
+        { country_import_id: T.must(@country_import).id },
+      )
+      redirect_back_or_to(
+        country_imports_url,
+        alert: "Interruption of country import failed with #{e.class}, #{e.message}",
+      )
+    end
+
+    private
+
+    sig { returns(T.nilable(CountryImport)) }
+    def set_country_import
+      @country_import = begin
+        CountryImport.find(params[:id])
+      rescue
+        nil
+      end
+    end
+
+    sig { returns(Integer) }
+    def set_page
+      @page = params.fetch(:page, 0).to_i
+    end
+  end
+end

data/app/controllers/atlas_engine/graphql_controller.rb

@@ -0,0 +1,59 @@
+# typed: false
+# frozen_string_literal: true
+
+module AtlasEngine
+  class GraphqlController < ApplicationController
+    # If accessing from outside this domain, nullify the session
+    # This allows for outside API access while preventing CSRF attacks,
+    # but you'll have to authenticate your user separately
+    protect_from_forgery with: :null_session, prepend: true
+
+    def execute
+      variables = prepare_variables(params[:variables])
+      query = params[:query]
+      operation_name = params[:operationName]
+      context = {
+        # Query context goes here, for example:
+        # current_user: current_user,
+      }
+      result = Schema.execute(query, variables: variables, context: context, operation_name: operation_name)
+      render(json: result)
+    rescue StandardError => e
+      raise e unless Rails.env.development?
+
+      handle_error_in_development(e)
+    end
+
+    private
+
+    # Handle variables in form data, JSON body, or a blank value
+    def prepare_variables(variables_param)
+      case variables_param
+      when String
+        if variables_param.present?
+          JSON.parse(variables_param) || {}
+        else
+          {}
+        end
+      when Hash
+        variables_param
+      when ActionController::Parameters
+        variables_param.to_unsafe_hash # GraphQL-Ruby will validate name and type of incoming variables.
+      when nil
+        {}
+      else
+        raise ArgumentError, "Unexpected parameter: #{variables_param}"
+      end
+    end
+
+    def handle_error_in_development(e)
+      logger.error(e.message)
+      logger.error(e.backtrace.join("\n"))
+
+      render(
+        json: { errors: [{ message: e.message, backtrace: e.backtrace }], data: {} },
+        status: :internal_server_error,
+      )
+    end
+  end
+end

data/app/countries/atlas_engine/ar/country_profile.yml

@@ -0,0 +1,9 @@
+id: AR
+ingestion:
+  settings:
+    min_zip_edge_ngram: "1"
+    max_zip_edge_ngram: "8"
+validation:
+  partial_postal_code_range_for_length:
+    4: 1..4
+    5: 0..4

data/app/countries/atlas_engine/at/address_importer/corrections/open_address/city_corrector.rb

@@ -0,0 +1,23 @@
+# typed: true
+# frozen_string_literal: true
+
+module AtlasEngine
+  module At
+    module AddressImporter
+      module Corrections
+        module OpenAddress
+          class CityCorrector
+            class << self
+              extend T::Sig
+
+              sig { params(address: Hash).void }
+              def apply(address)
+                address[:city] = ["Wien"] if address[:zip].starts_with?("1")
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/app/countries/atlas_engine/at/country_profile.yml

@@ -0,0 +1,24 @@
+id: AT
+ingestion:
+  correctors:
+    open_address:
+      - AtlasEngine::At::AddressImporter::Corrections::OpenAddress::CityCorrector
+  settings:
+    min_zip_edge_ngram: "1"
+    max_zip_edge_ngram: "4"
+  data_mapper: AtlasEngine::AddressValidation::Es::DataMappers::DecompoundingDataMapper
+validation:
+  enabled: true
+  default_matching_strategy: es
+  has_provinces: true
+  address_parser: AtlasEngine::At::ValidationTranscriber::AddressParser
+  normalized_components:
+  - region2
+  - region3
+  - region4
+  - city_aliases.alias
+  - suburb
+  - street_decompounded
+decompounding_patterns:
+  street:
+  - (?<name>\w+)(?<suffix>allee|gasse|kai|lande|pfad|platz|pl|ring|strasse|str|weg|zeile)(?:\b)

data/app/countries/atlas_engine/at/index_configuration.yml

@@ -0,0 +1,63 @@
+mappings:
+  properties:
+    street:
+      analyzer: street_indexing_analyzer
+      search_analyzer: street_decompounding_analyzer
+    street_stripped:
+      analyzer: street_indexing_analyzer
+      search_analyzer: street_decompounding_analyzer
+    street_decompounded:
+      type: text
+      analyzer: text_analyzer
+      fields:
+        keyword:
+          type: keyword
+settings:
+  index:
+    analysis:
+      analyzer:
+        text_analyzer:
+          filter:
+          - lowercase
+          - german_normalization
+          - icu_folding
+          - strip_special_characters
+        street_indexing_analyzer:
+          tokenizer: standard
+          filter:
+          - lowercase
+          - german_normalization
+          - icu_folding
+          - strip_special_characters
+          - street_suffix_decompounder
+        street_decompounding_analyzer:
+          tokenizer: standard
+          filter:
+          - lowercase
+          - german_normalization
+          - icu_folding
+          - strip_special_characters
+          - street_suffix_decompounder
+          - street_synonyms
+        street_analyzer:
+          filter:
+          - lowercase
+          - german_normalization
+          - icu_folding
+          - strip_special_characters
+          - street_synonyms
+        city_analyzer:
+          filter:
+          - lowercase
+          - german_normalization
+          - icu_folding
+          - strip_special_characters
+          - city_synonyms
+      filter:
+        street_suffix_decompounder:
+          type: pattern_capture
+          preserve_original: "false"
+          patterns:
+            <% country_profile.decompounding_patterns(:street).each do |pattern| %>
+            - <%= pattern %>
+            <% end %>

data/app/countries/atlas_engine/at/synonyms.yml

@@ -0,0 +1,6 @@
+street_synonyms:
+- platz, pl
+- strasse, str
+city_synonyms:
+- in, i
+- sankt, st

data/app/countries/atlas_engine/at/validation_transcriber/address_parser.rb

@@ -0,0 +1,58 @@
+# typed: true
+# frozen_string_literal: true
+
+module AtlasEngine
+  module At
+    module ValidationTranscriber
+      class AddressParser < AtlasEngine::ValidationTranscriber::AddressParserBase
+        private
+
+        STREET = "(?<street>.+?)"
+        NUMBERED_STREET = "(?<street>.+\s+[0-9]+)"
+        BUILDING_NUM = "(?<building_num>[0-9]+[[:alpha:]]*)"
+        UNIT_NUM = "(?<unit_num>.+)"
+
+        sig { returns(T::Array[Regexp]) }
+        def country_regex_formats
+          @country_regex_formats ||= [
+            %r{^#{STREET},?\s+#{BUILDING_NUM}([\s,-/]+#{UNIT_NUM})?$},
+            %r{^#{NUMBERED_STREET},?\s+#{BUILDING_NUM}([\s,-/]+#{UNIT_NUM})?$},
+          ]
+        end
+
+        sig { override.params(address_line: String).returns(T::Array[T.nilable(String)]) }
+        def extract_po_box(address_line)
+          [address_line, nil]
+        end
+
+        # Return true if something's obviously wrong with this regex match
+        sig do
+          override.params(
+            captures: T::Hash[Symbol, T.nilable(String)],
+            address: AddressValidation::TAddress,
+          ).returns(T::Boolean)
+        end
+        def ridiculous?(captures, address)
+          street = captures[:street]&.downcase
+
+          if street.present?
+            true unless address.address1&.upcase&.include?(street.upcase) ||
+              address.address2&.upcase&.include?(street.upcase)
+          end
+
+          false
+        end
+
+        sig { override.params(token: T.nilable(String)).returns(T::Boolean) }
+        def po_box?(token)
+          false
+        end
+
+        sig { override.params(token: T.nilable(String)).returns(T::Boolean) }
+        def street_suffix?(token)
+          false
+        end
+      end
+    end
+  end
+end

data/app/countries/atlas_engine/au/address_importer/open_address/filter.rb

@@ -0,0 +1,26 @@
+# typed: true
+# frozen_string_literal: true
+
+module AtlasEngine
+  module Au
+    module AddressImporter
+      module OpenAddress
+        class Filter
+          extend T::Sig
+          include AtlasEngine::AddressImporter::OpenAddress::Filter
+
+          def initialize(country_import:); end
+
+          sig { override.params(feature: AtlasEngine::AddressImporter::OpenAddress::Feature).returns(T::Boolean) }
+          def filter(feature)
+            # Discard features from the overseas territories, like Cocos, Christmas Island, etc.
+            # See: https://github.com/Shopify/worldwide/blob/c76c344/db/data/world.yml#L304-L308
+            return false if feature["region"] == "OT"
+
+            true
+          end
+        end
+      end
+    end
+  end
+end

data/app/countries/atlas_engine/au/address_importer/open_address/mapper.rb

@@ -0,0 +1,41 @@
+# typed: true
+# frozen_string_literal: true
+
+module AtlasEngine
+  module Au
+    module AddressImporter
+      module OpenAddress
+        class Mapper < AtlasEngine::AddressImporter::OpenAddress::DefaultMapper
+          sig do
+            params(feature: AtlasEngine::AddressImporter::OpenAddress::Feature).returns(T::Hash[Symbol, T.untyped])
+          end
+          def map(feature)
+            region, city, street, number, unit, postcode = feature["properties"].values_at(
+              "region",
+              "city",
+              "street",
+              "number",
+              "unit",
+              "postcode",
+            )
+            {
+              source_id: openaddress_source_id(feature),
+              locale: @locale,
+              country_code: "AU",
+              province_code: province_from_code(region),
+              # Omitted: region1..4
+              city: [city.titleize],
+              suburb: nil, # District field seems to always be empty and not useful
+              zip: postcode,
+              street: street.titleize,
+              # NOTE: unit seems to be 'UNIT|SUITE|CARSPACE|SHOP|VILLA N'. Do we strip text before the number?
+              building_and_unit_ranges: housenumber_and_unit(number, unit),
+              latitude: geometry(feature)&.at(1),
+              longitude: geometry(feature)&.at(0),
+            }
+          end
+        end
+      end
+    end
+  end
+end

data/app/countries/atlas_engine/au/country_profile.yml

@@ -0,0 +1,13 @@
+id: AU
+ingestion:
+  settings:
+    min_zip_edge_ngram: "1"
+    max_zip_edge_ngram: "4"
+  open_address:
+    feature_mapper: AtlasEngine::Au::AddressImporter::OpenAddress::Mapper
+validation:
+  address_parser: AtlasEngine::Au::ValidationTranscriber::AddressParser
+  enabled: true
+  default_matching_strategy: es
+open_address:
+  filter: AtlasEngine::Au::AddressImporter::OpenAddress::Filter