atlas_engine 0.1.0

data/app/countries/atlas_engine/gb/validation_transcriber/full_address_parser.rb

@@ -0,0 +1,164 @@
+# typed: strict
+# frozen_string_literal: true
+
+module AtlasEngine
+  module Gb
+    module ValidationTranscriber
+      class FullAddressParser
+        extend T::Sig
+        include AtlasEngine::ValidationTranscriber::Formatter
+
+        sig { params(address: AtlasEngine::AddressValidation::AbstractAddress).void }
+        def initialize(address:)
+          @address = address
+        end
+
+        # Note that parse() returns an array of possible interpretations, because it's sometimes
+        # impossible to be sure how to classify some parts of an address without more context.
+        #
+        # Consider the following two examples.
+        #
+        # Example 1:
+        #   2 Elm Avenue         [building_num] [dependent_street]
+        #   Runcorn Road         [street]
+        #   BIRMINGHAM           [post_town]
+        #   B12 8QX              [zip]
+        #
+        # Example 2:
+        #   1 Liechrhyd Terrace  [building_num] [street]
+        #   Builth Road          [dependent_locality]
+        #   BUILTH WELLS         [post_town]
+        #   LD2 3PY              [zip]
+        #
+        # In the first example, "Runcorn Road" is thoroughfare, and "Elm Avenue" is a dependent thoroughfare.
+        # In the second example, "Builth Road" is a dependent locality (hamlet), not a thoroughfare.
+        #
+        sig { returns(T::Array[ValidationTranscriber::ParsedAddress]) }
+        def parse
+          city = @address.city
+          if city.present?
+            city_parts = city.split(",").map(&:strip)
+            post_town = city_parts.first
+            if city_parts.count > 1
+              county = city_parts[1..-1]&.join(", ")
+            end
+          end
+
+          parsed_address = ValidationTranscriber::ParsedAddress.new(
+            zip: @address.zip,
+            province_code: @address.province_code,
+            country_code: @address.country_code || "GB",
+            post_town: post_town,
+            county: county,
+          )
+
+          # Split address1 and address2 on both commas and newlines
+          components = split_into_components(@address)
+
+          pivot = components.count > 3 ? 3 : components.length
+          pivot.downto(1).map do |pivot_value|
+            street_components, locality_components = components.partition.with_index do |_, index|
+              index <= components.count - pivot_value
+            end
+            hypothesize(
+              street_components: street_components,
+              locality_components: locality_components,
+              parsed_fields: parsed_address,
+            )
+          end.flatten
+        end
+
+        private
+
+        sig do
+          params(
+            conjecture: T::Hash[Symbol, String],
+            double_dependent_locality: T.nilable(String),
+            dependent_locality: T.nilable(String),
+            parsed_fields: ValidationTranscriber::ParsedAddress,
+          ).returns(ValidationTranscriber::ParsedAddress)
+        end
+        def address_from_conjecture(
+          conjecture:,
+          double_dependent_locality:,
+          dependent_locality:,
+          parsed_fields:
+        )
+          building_num = conjecture[:building_num]
+          unit_type = conjecture[:unit_type]
+          unit_num = conjecture[:unit_num]
+
+          if conjecture[:street].present?
+            street_parts = T.must(conjecture[:street]).split(",").map(&:strip)
+            if street_parts.count >= 2
+              dependent_street = street_parts.first
+              street = T.must(street_parts[1..-1]).join(", ")
+            else
+              street = street_parts.first
+            end
+          end
+
+          candidate = parsed_fields.dup
+
+          candidate.building_num = building_num if building_num.present?
+          candidate.dependent_street = dependent_street if dependent_street.present?
+          candidate.street = street if street.present?
+          candidate.unit_type = unit_type if unit_type.present?
+          candidate.unit_num = unit_num if unit_num.present?
+          candidate.double_dependent_locality = double_dependent_locality if double_dependent_locality.present?
+          candidate.dependent_locality = dependent_locality if dependent_locality.present?
+
+          candidate
+        end
+
+        sig do
+          params(
+            street_components: T::Array[String],
+            locality_components: T::Array[String],
+            parsed_fields: ValidationTranscriber::ParsedAddress,
+          ).returns(T::Array[ValidationTranscriber::ParsedAddress])
+        end
+        def hypothesize(street_components:, locality_components:, parsed_fields:)
+          double_dependent_locality = locality_components.first if locality_components.count >= 2
+          dependent_locality = locality_components.last if locality_components.count >= 1
+
+          address_line = street_components.join(", ")
+          conjectures = if parsed_fields.country_code.present?
+            AtlasEngine::ValidationTranscriber::AddressParserFactory.create(
+              address: build_address(
+                address1: address_line,
+                country_code: parsed_fields.country_code.to_s,
+              ),
+            ).parse
+          end
+
+          if conjectures.blank?
+            conjectures = [{ street: address_line }]
+          end
+
+          conjectures.map do |conjecture|
+            address_from_conjecture(
+              conjecture: conjecture,
+              double_dependent_locality: double_dependent_locality,
+              dependent_locality: dependent_locality,
+              parsed_fields: parsed_fields,
+            )
+          end
+        end
+
+        # Identify and return the "components" of the address
+        # These are portions of the address1 and/or address2 lines that have been separated by
+        # line break and/or by commas.
+        # For example,
+        #   123 High Street, Flat 4
+        #   Lower Hangleton, Swindon
+        # will return 4 components:
+        #   ["123 High Street", "Flat 4", "Lower Hangleton", "Swindon"]
+        sig { params(address: AtlasEngine::AddressValidation::AbstractAddress).returns(T::Array[String]) }
+        def split_into_components(address)
+          [address.address1&.split(","), address.address2&.split(",")].flatten.compact.map(&:strip)
+        end
+      end
+    end
+  end
+end

data/app/countries/atlas_engine/gb/validation_transcriber/parsed_address.rb

@@ -0,0 +1,120 @@
+# typed: strict
+# frozen_string_literal: true
+
+module AtlasEngine
+  module Gb
+    module ValidationTranscriber
+      class ParsedAddress
+        extend T::Sig
+
+        sig { returns(T.nilable(String)) }
+        attr_accessor :building_num # may be alphanumeric, e.g. "2A"
+
+        sig { returns(T.nilable(String)) }
+        attr_accessor :dependent_street # what Royal Mail calls a dependent thoroughfare
+
+        sig { returns(T.nilable(String)) }
+        attr_accessor :street # what Royal Mail calls a thoroughfare
+
+        sig { returns(T.nilable(String)) }
+        attr_accessor :unit_type
+
+        sig { returns(T.nilable(String)) }
+        attr_accessor :unit_num
+
+        sig { returns(T.nilable(String)) }
+        attr_accessor :double_dependent_locality
+
+        sig { returns(T.nilable(String)) }
+        attr_accessor :dependent_locality
+
+        sig { returns(T.nilable(String)) }
+        attr_accessor :post_town # the :city field may contain "{post_town}, {county}"
+
+        sig { returns(T.nilable(String)) }
+        attr_accessor :county # deprecated in 1996, but still permitted
+
+        sig { returns(T.nilable(String)) }
+        attr_accessor :province_code
+
+        sig { returns(T.any(Symbol, String)) }
+        attr_accessor :country_code
+
+        sig { returns(T.nilable(String)) }
+        attr_accessor :zip
+
+        sig do
+          params(
+            country_code: T.any(Symbol, String),
+            building_num: T.nilable(String),
+            dependent_street: T.nilable(String),
+            street: T.nilable(String),
+            unit_type: T.nilable(String),
+            unit_num: T.nilable(String),
+            double_dependent_locality: T.nilable(String),
+            dependent_locality: T.nilable(String),
+            post_town: T.nilable(String),
+            county: T.nilable(String),
+            province_code: T.nilable(String),
+            zip: T.nilable(String),
+          ).void
+        end
+        def initialize(
+          country_code:,
+          building_num: nil,
+          dependent_street: nil,
+          street: nil,
+          unit_type: nil,
+          unit_num: nil,
+          double_dependent_locality: nil,
+          dependent_locality: nil,
+          post_town: nil,
+          county: nil,
+          province_code: nil,
+          zip: nil
+        )
+          @building_num = building_num
+          @dependent_street = dependent_street
+          @street = street
+          @unit_type = unit_type
+          @unit_num = unit_num
+          @double_dependent_locality = double_dependent_locality
+          @dependent_locality = dependent_locality
+          @post_town = post_town
+          @county = county
+          @province_code = province_code
+          @country_code = country_code
+          @zip = zip
+        end
+
+        sig { params(other: ParsedAddress).returns(T::Boolean) }
+        def ==(other)
+          @country_code.presence == other.country_code.presence &&
+            @building_num.presence == other.building_num.presence &&
+            @dependent_street.presence == other.dependent_street.presence &&
+            @street.presence == other.street.presence &&
+            @unit_type.presence == other.unit_type.presence &&
+            @unit_num.presence == other.unit_num.presence &&
+            @double_dependent_locality.presence == other.double_dependent_locality.presence &&
+            @dependent_locality.presence == other.dependent_locality.presence &&
+            @post_town.presence == other.post_town.presence &&
+            @county.presence == other.county.presence &&
+            @zip.presence == other.zip.presence &&
+
+            # If no province is given in GB, we'll infer it with a postcode lookup
+            (
+              @province_code.presence == other.province_code.presence ||
+                inferred_province_code == other.send(:inferred_province_code)
+            )
+        end
+
+        private
+
+        sig { returns(T.nilable(String)) }
+        def inferred_province_code
+          @province_code.presence || Worldwide.region(code: @country_code)&.zone(zip: @zip)&.legacy_code
+        end
+      end
+    end
+  end
+end

data/app/countries/atlas_engine/gg/address_validation/validators/full_address/restrictions/unsupported_city.rb

@@ -0,0 +1,39 @@
+# typed: true
+# frozen_string_literal: true
+
+module AtlasEngine
+  module Gg
+    module AddressValidation
+      module Validators
+        module FullAddress
+          module Restrictions
+            class UnsupportedCity
+              UNSUPPORTED_CITY_ZIP_MAPPING = {
+                "SARK" => "GY9",
+                "ALDERNEY" => "GY10",
+              }.freeze
+
+              class << self
+                extend T::Sig
+                include AtlasEngine::Restrictions::Base
+
+                sig do
+                  override.params(
+                    address: AtlasEngine::AddressValidation::AbstractAddress,
+                    params: T.untyped,
+                  ).returns(T::Boolean)
+                end
+                def apply?(address:, params: nil)
+                  zip_prefix = UNSUPPORTED_CITY_ZIP_MAPPING[address.city&.upcase]
+                  return false if zip_prefix.nil?
+
+                  address.zip&.start_with?(zip_prefix).present?
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/app/countries/atlas_engine/gg/country_profile.yml

@@ -0,0 +1,7 @@
+id: GG
+validation:
+  enabled: true
+  default_matching_strategy: es
+  has_provinces: false
+  restrictions:
+    - class: AtlasEngine::Gg::AddressValidation::Validators::FullAddress::Restrictions::UnsupportedCity

data/app/countries/atlas_engine/ie/country_profile.yml

@@ -0,0 +1,3 @@
+id: IE
+ingestion:
+  allow_partial_zip?: true

data/app/countries/atlas_engine/it/address_importer/corrections/open_address/city_corrector.rb

@@ -0,0 +1,27 @@
+# typed: true
+# frozen_string_literal: true
+
+module AtlasEngine
+  module It
+    module AddressImporter
+      module Corrections
+        module OpenAddress
+          class CityCorrector
+            class << self
+              extend T::Sig
+
+              sig { params(address: Hash).void }
+              def apply(address)
+                if address[:city] == ["Sissa"]
+                  address[:city] = ["Sissa Trecasali"]
+                elsif address[:city] == ["Reggio Nell'emilia"]
+                  address[:city] << "Reggio Emilia"
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/app/countries/atlas_engine/it/address_importer/corrections/open_address/province_corrector.rb

@@ -0,0 +1,29 @@
+# typed: true
+# frozen_string_literal: true
+
+module AtlasEngine
+  module It
+    module AddressImporter
+      module Corrections
+        module OpenAddress
+          class ProvinceCorrector
+            class << self
+              extend T::Sig
+
+              sig { params(address: Hash).void }
+              def apply(address)
+                if address[:region2] == "BOLZANO/BOZEN"
+                  address[:province_code] = "BZ"
+                elsif address[:region2] == "FORLI'-CESENA"
+                  address[:province_code] = "FC"
+                elsif address[:region2] == "REGGIO DI CALABRIA"
+                  address[:province_code] = "RC"
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/app/countries/atlas_engine/it/address_importer/open_address/mapper.rb

@@ -0,0 +1,42 @@
+# typed: true
+# frozen_string_literal: true
+
+module AtlasEngine
+  module It
+    module AddressImporter
+      module OpenAddress
+        class Mapper < AtlasEngine::AddressImporter::OpenAddress::DefaultMapper
+          sig do
+            params(feature: AtlasEngine::AddressImporter::OpenAddress::Feature).returns(T::Hash[Symbol, T.untyped])
+          end
+          def map(feature)
+            region, district, city, street, number, unit, postcode = feature["properties"].values_at(
+              "region",
+              "district",
+              "city",
+              "street",
+              "number",
+              "unit",
+              "postcode",
+            )
+            {
+              source_id: openaddress_source_id(feature),
+              locale: @locale,
+              country_code: "IT",
+              province_code: province_code_from_name(district),
+              region1: region,
+              region2: district,
+              city: [city.titleize],
+              suburb: nil,
+              zip: postcode,
+              street: street.titleize,
+              building_and_unit_ranges: housenumber_and_unit(number, unit),
+              latitude: geometry(feature)&.at(1),
+              longitude: geometry(feature)&.at(0),
+            }
+          end
+        end
+      end
+    end
+  end
+end

data/app/countries/atlas_engine/it/country_profile.yml

@@ -0,0 +1,11 @@
+id: IT
+validation:
+  enabled: true
+  default_matching_strategy: es
+ingestion:
+  open_address:
+    feature_mapper: AtlasEngine::It::AddressImporter::OpenAddress::Mapper
+  correctors:
+    open_address:
+      - AtlasEngine::It::AddressImporter::Corrections::OpenAddress::CityCorrector
+      - AtlasEngine::It::AddressImporter::Corrections::OpenAddress::ProvinceCorrector

data/app/countries/atlas_engine/jp/address_validation/es/data_mapper.rb

@@ -0,0 +1,63 @@
+# typed: strict
+# frozen_string_literal: true
+
+module AtlasEngine
+  module Jp
+    module AddressValidation
+      module Es
+        class DataMapper < AtlasEngine::AddressValidation::Es::DataMappers::DefaultDataMapper
+          sig do
+            returns(T::Hash[Symbol, T.untyped])
+          end
+          def map_data
+            if T.must(locale_language_code.casecmp("en")).zero?
+              super.merge(en_modifications)
+            else
+              super.merge(ja_modifications)
+            end
+          end
+
+          private
+
+          sig { returns(T::Hash[T.untyped, T.untyped]) }
+          def en_modifications
+            {
+              region3: mapped_region_3,
+              city_aliases: city_aliases(mapped_cities),
+              street: format_en_street(
+                components: [
+                  mapped_region_3.gsub("Others", ""),
+                  post_address[:region4],
+                ],
+              ),
+            }
+          end
+
+          sig { returns(T::Hash[T.untyped, T.untyped]) }
+          def ja_modifications
+            {
+              region3: mapped_region_3,
+              city_aliases: city_aliases(mapped_cities),
+              street: "#{post_address[:region4]}#{mapped_region_3.gsub("その他", "")}",
+            }
+          end
+
+          sig { returns(String) }
+          def mapped_region_3
+            post_address[:city].first
+          end
+
+          sig { returns(T::Array[String]) }
+          def mapped_cities
+            [post_address[:region3]]
+          end
+
+          sig { params(components: T::Array[T.nilable(String)]).returns(T.nilable(String)) }
+          def format_en_street(components:)
+            Worldwide.lists.format(components.compact.reject(&:empty?), join: :narrow)
+          end
+        end
+      end
+    end
+  end
+end

data/app/countries/atlas_engine/jp/country_profile.yml

@@ -0,0 +1,6 @@
+id: JP
+ingestion:
+  data_mapper: AtlasEngine::Jp::AddressValidation::Es::DataMapper
+  settings:
+    min_zip_edge_ngram: "3"
+    max_zip_edge_ngram: "8"

data/app/countries/atlas_engine/kr/address_importer/open_address/mapper.rb

@@ -0,0 +1,41 @@
+# typed: true
+# frozen_string_literal: true
+
+module AtlasEngine
+  module Kr
+    module AddressImporter
+      module OpenAddress
+        class Mapper < AtlasEngine::AddressImporter::OpenAddress::DefaultMapper
+          sig do
+            params(feature: AtlasEngine::AddressImporter::OpenAddress::Feature).returns(T::Hash[Symbol, T.untyped])
+          end
+          def map(feature)
+            region, district, city, street, number, unit, postcode = feature["properties"].values_at(
+              "region",
+              "district",
+              "city",
+              "street",
+              "number",
+              "unit",
+              "postcode",
+            )
+            {
+              source_id: openaddress_source_id(feature),
+              locale: @locale,
+              country_code: "KR",
+              province_code: province_code_from_name(region),
+              region1: region,
+              city: [city],
+              suburb: district,
+              zip: postcode,
+              street: street,
+              building_and_unit_ranges: housenumber_and_unit(number, unit),
+              latitude: geometry(feature)&.at(1),
+              longitude: geometry(feature)&.at(0),
+            }
+          end
+        end
+      end
+    end
+  end
+end

data/app/countries/atlas_engine/kr/country_profile.yml

@@ -0,0 +1,11 @@
+id: KR
+ingestion:
+  open_address:
+    feature_mapper: AtlasEngine::Kr::AddressImporter::OpenAddress::Mapper
+validation:
+  enabled: true
+  default_matching_strategy: es
+  restrictions:
+    - class: AtlasEngine::Restrictions::UnsupportedScript
+      params:
+        supported_script: :Hangul

data/app/countries/atlas_engine/li/address_importer/corrections/open_address/city_corrector.rb

@@ -0,0 +1,25 @@
+# typed: true
+# frozen_string_literal: true
+
+module AtlasEngine
+  module Li
+    module AddressImporter
+      module Corrections
+        module OpenAddress
+          class CityCorrector
+            class << self
+              extend T::Sig
+
+              sig { params(address: Hash).void }
+              def apply(address)
+                if address[:city].include?("Gamprin-Bendern")
+                  address[:city] += ["Gamprin", "Bendern"]
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/app/countries/atlas_engine/li/country_profile.yml

@@ -0,0 +1,21 @@
+id: LI
+ingestion:
+  correctors:
+    open_address:
+      - AtlasEngine::Li::AddressImporter::Corrections::OpenAddress::CityCorrector
+  data_mapper: AtlasEngine::AddressValidation::Es::DataMappers::DecompoundingDataMapper
+validation:
+  enabled: true
+  has_provinces: true
+  default_matching_strategy: es
+  address_parser: AtlasEngine::At::ValidationTranscriber::AddressParser
+  normalized_components:
+  - region2
+  - region3
+  - region4
+  - city_aliases.alias
+  - suburb
+  - street_decompounded
+decompounding_patterns:
+  street:
+  - (?<name>\w+)(?<suffix>allee|gasse|kai|lande|pfad|platz|pl|ring|strasse|str|weg|zeile)(?:\b)

data/app/countries/atlas_engine/li/index_configuration.yml

@@ -0,0 +1,63 @@
+mappings:
+  properties:
+    street:
+      analyzer: street_indexing_analyzer
+      search_analyzer: street_decompounding_analyzer
+    street_stripped:
+      analyzer: street_indexing_analyzer
+      search_analyzer: street_decompounding_analyzer
+    street_decompounded:
+      type: text
+      analyzer: text_analyzer
+      fields:
+        keyword:
+          type: keyword
+settings:
+  index:
+    analysis:
+      analyzer:
+        text_analyzer:
+          filter:
+          - lowercase
+          - german_normalization
+          - icu_folding
+          - strip_special_characters
+        street_indexing_analyzer:
+          tokenizer: standard
+          filter:
+          - lowercase
+          - german_normalization
+          - icu_folding
+          - strip_special_characters
+          - street_suffix_decompounder
+        street_decompounding_analyzer:
+          tokenizer: standard
+          filter:
+          - lowercase
+          - german_normalization
+          - icu_folding
+          - strip_special_characters
+          - street_suffix_decompounder
+          - street_synonyms
+        street_analyzer:
+          filter:
+          - lowercase
+          - german_normalization
+          - icu_folding
+          - strip_special_characters
+          - street_synonyms
+        city_analyzer:
+          filter:
+          - lowercase
+          - german_normalization
+          - icu_folding
+          - strip_special_characters
+          - city_synonyms
+      filter:
+        street_suffix_decompounder:
+          type: pattern_capture
+          preserve_original: "false"
+          patterns:
+            <% country_profile.decompounding_patterns(:street).each do |pattern| %>
+            - <%= pattern %>
+            <% end %>