atlas_engine 0.1.0

data/app/lib/atlas_engine/validation_transcriber/french_street_parser.rb

@@ -0,0 +1,50 @@
+# typed: false
+# frozen_string_literal: true
+
+module AtlasEngine
+  module ValidationTranscriber
+    class FrenchStreetParser
+      include AddressParsingHelper
+
+      def initialize
+        super
+      end
+
+      def parse(street:)
+        return {} if street.blank?
+
+        # Expected format: [suffix, pre_directional, name, post_directional]
+        # Note that the directionals may be absent.
+        # Suffix is actually a prefix in French, but we keep the existing terminology for cross-language consistency.
+
+        suffix = nil
+        pre_directional = nil
+        post_directional = nil
+
+        tokens = street.split(" ")
+
+        if street_suffix?(tokens[0])
+          suffix = tokens[0]
+          tokens = tokens[1..-1]
+        end
+
+        if directional?(tokens[0])
+          pre_directional = tokens[0]
+          tokens = tokens[1..-1]
+        end
+
+        if directional?(tokens[-1])
+          post_directional = tokens[-1]
+          tokens = tokens[0..-2]
+        end
+
+        {
+          pre_directional: pre_directional,
+          name: tokens.join(" "),
+          suffix: suffix,
+          post_directional: post_directional,
+        }.compact_blank.to_h
+      end
+    end
+  end
+end

data/app/lib/atlas_engine/validation_transcriber/province_code_normalizer.rb

@@ -0,0 +1,45 @@
+# typed: strict
+# frozen_string_literal: true
+
+module AtlasEngine
+  module ValidationTranscriber
+    class ProvinceCodeNormalizer
+      class << self
+        extend T::Sig
+        sig { params(country_code: T.nilable(String), province_code: T.nilable(String)).returns(T.nilable(String)) }
+        def normalize(country_code:, province_code:)
+          return if province_code.blank?
+          return province_code if country_code.blank?
+
+          iso_code(country_code, province_code) ||
+            iso_from_cldr(country_code, province_code) ||
+            province_code
+        end
+
+        private
+
+        sig { params(country_code: String, province_code: String).returns(T.nilable(String)) }
+        def iso_code(country_code, province_code)
+          zone = Worldwide.region(code: country_code)&.zone(code: province_code)
+          zone.province? ? zone.iso_code : nil
+        end
+
+        sig { params(country_code: String, province_code: String).returns(T.nilable(String)) }
+        def iso_from_cldr(country_code, province_code)
+          zone = Worldwide.region(code: province_code)
+          if zone_valid?(zone, country_code)
+            return zone.iso_code
+          end
+
+          zone = Worldwide.region(cldr: province_code)
+          zone_valid?(zone, country_code) ? zone.iso_code : nil
+        end
+
+        sig { params(zone: Worldwide::Region, country_code: String).returns(T::Boolean) }
+        def zone_valid?(zone, country_code)
+          zone.province? && zone.associated_country.iso_code.casecmp(country_code) == 0
+        end
+      end
+    end
+  end
+end

data/app/lib/atlas_engine/validation_transcriber/street_parser.rb

@@ -0,0 +1,18 @@
+# typed: false
+# frozen_string_literal: true
+
+module AtlasEngine
+  module ValidationTranscriber
+    class StreetParser
+      def parse(street:, locale: nil)
+        lang = Worldwide.locale(code: locale || I18n.locale).language_subtag.to_s.downcase
+
+        if "fr" == lang
+          FrenchStreetParser.new.parse(street: street)
+        else
+          EnglishStreetParser.new.parse(street: street)
+        end
+      end
+    end
+  end
+end

data/app/lib/atlas_engine/validation_transcriber/zip_normalizer.rb

@@ -0,0 +1,23 @@
+# typed: strict
+# frozen_string_literal: true
+
+module AtlasEngine
+  module ValidationTranscriber
+    class ZipNormalizer
+      class << self
+        extend T::Sig
+
+        sig { params(country_code: T.nilable(String), zip: T.nilable(String)).returns(T.nilable(String)) }
+        def normalize(country_code:, zip:)
+          if country_code.present? && Worldwide.region(code: country_code).valid_zip?(zip)
+            AddressValidation::ZipTruncator.new(country_code: country_code).truncate(
+              zip: Worldwide::Zip.normalize(country_code: country_code, zip: zip, strip_extraneous_characters: true),
+            )
+          else
+            Worldwide::Zip.normalize(country_code: country_code, zip: zip)
+          end
+        end
+      end
+    end
+  end
+end

data/app/mailers/atlas_engine/application_mailer.rb

@@ -0,0 +1,9 @@
+# typed: false
+# frozen_string_literal: true
+
+module AtlasEngine
+  class ApplicationMailer < ActionMailer::Base
+    default from: "from@example.com"
+    layout "mailer"
+  end
+end

data/app/models/atlas_engine/address_importer/corrections/corrector.rb

@@ -0,0 +1,33 @@
+# typed: true
+# frozen_string_literal: true
+
+module AtlasEngine
+  module AddressImporter
+    module Corrections
+      class Corrector
+        extend T::Sig
+        attr_reader :correctors
+
+        sig { params(country_code: String, source: String).void }
+        def initialize(country_code:, source:)
+          @country_code = country_code.upcase
+          @correctors ||= correctors_for_country(@country_code, source)
+        end
+
+        sig { params(address: Hash).void }
+        def apply(address)
+          correctors.each do |corrector|
+            corrector.apply(address)
+          end
+        end
+
+        private
+
+        sig { params(country_code: String, source: String).returns(T::Array[Class]) }
+        def correctors_for_country(country_code, source)
+          CountryProfile.for(country_code).ingestion.correctors(source: source)
+        end
+      end
+    end
+  end
+end

data/app/models/atlas_engine/address_importer/import_events_notifier/base.rb

@@ -0,0 +1,35 @@
+# typed: strict
+# frozen_string_literal: true
+
+module AtlasEngine
+  module AddressImporter
+    module ImportEventsNotifier
+      class Base
+        extend T::Sig
+        extend T::Helpers
+        abstract!
+
+        class << self
+          extend T::Sig
+
+          sig { returns(T.untyped) }
+          def instance
+            T.unsafe(self).new
+          end
+        end
+
+        sig { params(client: T.untyped).void }
+        def initialize(client: nil)
+          @client = client
+        end
+
+        sig do
+          abstract.params(
+            event: Event,
+          ).void
+        end
+        def notify(event:); end
+      end
+    end
+  end
+end

data/app/models/atlas_engine/address_importer/import_events_notifier/notifier.rb

@@ -0,0 +1,26 @@
+# typed: true
+# frozen_string_literal: true
+
+module AtlasEngine
+  module AddressImporter
+    module ImportEventsNotifier
+      class Notifier < Base
+        extend T::Sig
+        extend T::Helpers
+
+        include Singleton
+
+        sig do
+          override.params(
+            event: Event,
+          ).void
+        end
+        def notify(event:)
+          # do nothing,
+          # The Host application can define its own notifier by configuring
+          # AtlasEngine.address_importer_notifier = MyCustomNotifier
+        end
+      end
+    end
+  end
+end

data/app/models/atlas_engine/address_importer/open_address/default_mapper.rb

@@ -0,0 +1,46 @@
+# typed: true
+# frozen_string_literal: true
+
+module AtlasEngine
+  module AddressImporter
+    module OpenAddress
+      class DefaultMapper
+        extend T::Sig
+        include FeatureHelper
+        sig { params(country_code: String, locale: T.nilable(String)).void }
+        def initialize(country_code:, locale: nil)
+          @country_code = country_code
+          @locale = locale
+        end
+
+        sig { params(feature: Feature).returns(T::Hash[Symbol, T.untyped]) }
+        def map(feature)
+          region, city, street, number, unit, postcode = feature["properties"].values_at(
+            "region",
+            "city",
+            "street",
+            "number",
+            "unit",
+            "postcode",
+          )
+          {
+            source_id: openaddress_source_id(feature),
+            locale: @locale,
+            country_code: @country_code,
+            province_code: nil,
+            region1: region,
+            # Don't titleize. The sources have proper capitalization, and it's a problem for cities like
+            # 's-Graveland, which would get titleized to "'S Graveland" which is wrong.
+            city: [city],
+            suburb: nil,
+            zip: normalize_zip(postcode),
+            street: street,
+            building_and_unit_ranges: housenumber_and_unit(number, unit),
+            latitude: geometry(feature)&.at(1),
+            longitude: geometry(feature)&.at(0),
+          }
+        end
+      end
+    end
+  end
+end

data/app/models/atlas_engine/address_importer/open_address/feature_helper.rb

@@ -0,0 +1,110 @@
+# typed: true
+# frozen_string_literal: true
+
+module AtlasEngine
+  module AddressImporter
+    module OpenAddress
+      module FeatureHelper
+        extend T::Sig
+        extend T::Helpers
+
+        sig { params(feature: T::Hash[String, T.untyped]).returns(T.nilable(String)) }
+        def openaddress_source_id(feature)
+          objid, hash = feature["properties"].values_at("id", "hash")
+          if objid.present?
+            # OA indicates an OpenAddresses-provided ID
+            "OA-#{objid}"
+          elsif hash.present?
+            # Which may come from a different field, hence the hash sign
+            "OA##{hash}"
+          else
+            # AT signifies an Atlas-calculated hash
+            "AT-#{signature(feature)}"
+          end
+        end
+
+        sig do
+          params(number: T.nilable(String), unit: T.nilable(String))
+            .returns(T.nilable(T::Hash[T.untyped, T.untyped]))
+        end
+        def housenumber_and_unit(number, unit)
+          return {} if number.blank?
+
+          { number => unit.present? ? { unit => {} } : {} }
+        end
+
+        sig { params(feature: Feature).returns(T.nilable([Numeric, Numeric])) }
+        def geometry(feature)
+          geom = feature["geometry"]
+          if geom.present? && geom["type"] == "Point"
+            geom["coordinates"]
+          end
+        end
+
+        sig { params(zip: String).returns(String) }
+        def normalize_zip(zip)
+          Worldwide::Zip.normalize(
+            country_code: @country_code,
+            zip: zip,
+            strip_extraneous_characters: true,
+          )
+        end
+
+        sig { params(district: String).returns(T.nilable(String)) }
+        def province_code_from_name(district) = zones_mapping[district.downcase]
+
+        sig { params(region: String).returns(T.nilable(String)) }
+        def province_from_code(region) = zone_codes.include?(region) ? region : nil
+
+        sig { params(zip: String).returns(T.nilable(String)) }
+        def province_code_from_zip(zip)
+          @zip_to_province_mapping ||= {}
+          unless @zip_to_province_mapping.key?(zip)
+            province = Worldwide.region(code: @country_code).zone(zip: zip)
+            @zip_to_province_mapping[zip] = province.province? ? province.iso_code : nil
+          end
+          @zip_to_province_mapping[zip]
+        end
+
+        private
+
+        sig { params(feature: T::Hash[String, T.untyped]).returns(String) }
+        def signature(feature)
+          Digest::MD5.hexdigest(
+            feature["properties"].map { "#{_1}=#{_2}" }.join("\n"),
+          )
+        end
+
+        sig { returns(T::Hash[String, String]) }
+        def zones_mapping
+          @zones_mapping ||= legacy_zone_names.merge(full_zone_names).merge(name_alternates)
+        end
+
+        sig { returns(T::Array[Worldwide::Region]) }
+        def zones = @zones ||= Worldwide.region(code: @country_code).zones
+
+        sig { returns(T::Hash[String, String]) }
+        def legacy_zone_names
+          zones.to_h { |z| [z.legacy_name.downcase, z.legacy_code] }
+        end
+
+        sig { returns(T::Hash[String, String]) }
+        def full_zone_names
+          zones.to_h { |z| [z.full_name.downcase, z.legacy_code] }
+        end
+
+        sig { returns(T::Hash[String, String]) }
+        def name_alternates
+          zones_with_alternates = zones.select { |z| z.name_alternates.present? }
+          zones_with_alternates.each_with_object({}) do |zone, hash|
+            zone.name_alternates.each { |name| hash[name.downcase] = zone.legacy_code }
+          end
+        end
+
+        def zone_codes
+          @zone_codes ||= Worldwide.region(code: @country_code).zones.to_set(&:legacy_code)
+        end
+      end
+    end
+  end
+end

data/app/models/atlas_engine/address_importer/open_address/filter.rb

@@ -0,0 +1,17 @@
+# typed: true
+# frozen_string_literal: true
+
+module AtlasEngine
+  module AddressImporter
+    module OpenAddress
+      module Filter
+        extend T::Sig
+        extend T::Helpers
+        interface!
+
+        sig { abstract.params(feature: T::Hash[String, T.untyped]).returns(T::Boolean) }
+        def filter(feature); end
+      end
+    end
+  end
+end

data/app/models/atlas_engine/address_importer/open_address/loader.rb

@@ -0,0 +1,27 @@
+# typed: true
+# frozen_string_literal: true
+
+module AtlasEngine
+  module AddressImporter
+    module OpenAddress
+      class Loader
+        extend T::Sig
+
+        sig { params(addresses: T::Array[Hash]).void }
+        def load(addresses)
+          PostAddress.upsert_all( # rubocop:disable Rails/SkipsModelValidations
+            addresses,
+            on_duplicate: merge_building_ranges_clause,
+          )
+        end
+
+        private
+
+        def merge_building_ranges_clause
+          Arel.sql("building_and_unit_ranges =
+              JSON_MERGE(#{PostAddress.table_name}.building_and_unit_ranges, VALUES(building_and_unit_ranges))")
+        end
+      end
+    end
+  end
+end

data/app/models/atlas_engine/address_importer/open_address/transformer.rb

@@ -0,0 +1,39 @@
+# typed: true
+# frozen_string_literal: true
+
+module AtlasEngine
+  module AddressImporter
+    module OpenAddress
+      class Transformer
+        extend T::Sig
+        extend T::Helpers
+
+        def initialize(country_import:, locale: nil)
+          @country_code = country_import.country_code
+          @locale = locale
+          @mapper = CountryProfile.for(@country_code).ingestion.open_address_feature_mapper.new(
+            country_code: @country_code, locale: @locale,
+          )
+          @corrector = AddressImporter::Corrections::Corrector.new(country_code: @country_code, source: "open_address")
+          @validator = AddressImporter::Validation::Wrapper.new(
+            country_import: country_import,
+            log_invalid_records: false,
+          )
+        end
+
+        sig do
+          params(feature: Feature)
+            .returns(T.nilable(T::Hash[Symbol, T.untyped]))
+        end
+        def transform(feature)
+          address_hash = @mapper.map(feature)
+
+          @corrector.apply(address_hash)
+          return if address_hash.blank?
+
+          address_hash if @validator.valid?(address_hash)
+        end
+      end
+    end
+  end
+end

data/app/models/atlas_engine/address_importer/open_address.rb

@@ -0,0 +1,10 @@
+# typed: true
+# frozen_string_literal: true
+
+module AtlasEngine
+  module AddressImporter
+    module OpenAddress
+      Feature = T.type_alias { T::Hash[String, T.untyped] }
+    end
+  end
+end

data/app/models/atlas_engine/address_importer/validation/base_validator.rb

@@ -0,0 +1,86 @@
+# typed: true
+# frozen_string_literal: true
+
+module AtlasEngine
+  module AddressImporter
+    module Validation
+      class BaseValidator
+        extend T::Sig
+
+        Errors = T.type_alias { T::Hash[Symbol, T.untyped] }
+
+        attr_reader :allow_partial_zip, :field_validations, :additional_field_validations
+
+        sig do
+          params(
+            country_code: String,
+            field_validations: T::Hash[Symbol, T::Array[FieldValidations::Interface]],
+            additional_field_validations: T::Hash[Symbol, T::Array[FieldValidations::Interface]],
+          ).void
+        end
+        def initialize(country_code:, field_validations:, additional_field_validations: {})
+          @allow_partial_zip = T.let(CountryProfile.partial_zip_allowed_countries.include?(country_code), T::Boolean)
+          @field_validations = field_validations
+          @additional_field_validations = additional_field_validations.transform_values do |validator_classes|
+            validator_classes.map do |validator_class|
+              if validator_class.is_a?(Class)
+                validator_class
+              else
+                validator_class.to_s.constantize
+              end
+            end
+          end
+
+          @caches = {}
+          merged_field_validations.map do |field, _validator_class|
+            @caches[field] = Set.new
+          end
+
+          @errors = T.let(Hash.new([]), Errors)
+        end
+
+        sig { params(address: AddressImporter::Validation::Wrapper::AddressStruct).returns(Errors) }
+        def validation_errors(address:)
+          validate(address: address)
+          @errors
+        end
+
+        private
+
+        sig { returns(T::Hash[Symbol, T::Array[FieldValidations::Interface]]) }
+        def merged_field_validations
+          @merged_field_validations ||= @field_validations.merge(@additional_field_validations) do
+            |_field, validator_class, additional_validator_class|
+
+            [validator_class, additional_validator_class].flatten.compact.uniq
+          end
+        end
+
+        sig { params(address: AddressImporter::Validation::Wrapper::AddressStruct).void }
+        def validate(address:)
+          clear_errors
+
+          merged_field_validations.each do |field, validators|
+            if @caches[field].exclude?(address[field])
+              @errors[field] = [].tap do |error_msgs|
+                validators.each do |validator_class|
+                  validator = T.unsafe(validator_class).new(address: address, allow_partial_zip: allow_partial_zip)
+                  error_msgs << validator.validation_errors
+                end
+              end.flatten
+            end
+
+            break if @errors[field].present?
+
+            @caches[field].add(address[field])
+          end
+        end
+
+        sig { void }
+        def clear_errors
+          @errors = Hash.new([])
+        end
+      end
+    end
+  end
+end

data/app/models/atlas_engine/address_importer/validation/default_validator.rb

@@ -0,0 +1,27 @@
+# typed: true
+# frozen_string_literal: true
+
+module AtlasEngine
+  module AddressImporter
+    module Validation
+      class DefaultValidator < BaseValidator
+        extend T::Sig
+
+        sig { params(country_code: String).void }
+        def initialize(country_code:)
+          field_validations = {
+            city: [FieldValidations::City],
+            province_code: [FieldValidations::Province],
+            zip: [FieldValidations::Zip],
+          }
+
+          super(
+            country_code: country_code,
+            field_validations: field_validations,
+            additional_field_validations: AtlasEngine.address_importer_additional_field_validations,
+          )
+        end
+      end
+    end
+  end
+end

data/app/models/atlas_engine/address_importer/validation/field_validations/city.rb

@@ -0,0 +1,47 @@
+# typed: true
+# frozen_string_literal: true
+
+module AtlasEngine
+  module AddressImporter
+    module Validation
+      module FieldValidations
+        class City
+          extend T::Sig
+          extend T::Helpers
+          include Interface
+
+          sig do
+            override.params(
+              address: AddressImporter::Validation::Wrapper::AddressStruct,
+              allow_partial_zip: T::Boolean,
+            ).void
+          end
+          def initialize(address:, allow_partial_zip: false)
+            @city = address.city
+            @errors = []
+          end
+
+          sig { override.returns(T::Array[String]) }
+          def validation_errors
+            clear_errors
+            validate
+            errors
+          end
+
+          private
+
+          attr_reader :city
+          attr_accessor :errors
+
+          def clear_errors
+            self.errors = []
+          end
+
+          def validate
+            errors << "City is required" unless city&.any?(&:present?)
+          end
+        end
+      end
+    end
+  end
+end

data/app/models/atlas_engine/address_importer/validation/field_validations/interface.rb

@@ -0,0 +1,29 @@
+# typed: true
+# frozen_string_literal: true
+
+module AtlasEngine
+  module AddressImporter
+    module Validation
+      module FieldValidations
+        module Interface
+          extend T::Sig
+          extend T::Helpers
+          include Kernel
+
+          interface!
+
+          sig do
+            abstract.params(
+              address: AddressImporter::Validation::Wrapper::AddressStruct,
+              allow_partial_zip: T::Boolean,
+            ).void
+          end
+          def initialize(address:, allow_partial_zip: false); end
+
+          sig { abstract.returns(T::Array[String]) }
+          def validation_errors; end
+        end
+      end
+    end
+  end
+end