atlas_engine 0.1.0

data/app/countries/atlas_engine/ch/address_importer/corrections/open_address/city_corrector.rb

@@ -0,0 +1,29 @@
+# typed: true
+# frozen_string_literal: true
+
+module AtlasEngine
+  module Ch
+    module AddressImporter
+      module Corrections
+        module OpenAddress
+          class CityCorrector
+            class << self
+              extend T::Sig
+
+              sig { params(address: Hash).void }
+              def apply(address)
+                if address[:city] == ["Biel/Bienne"]
+                  address[:city] += ["Bienne", "Biel"]
+                elsif address[:city] == ["Emmenbrücke"]
+                  address[:city] << "Emmen"
+                elsif address[:city] == ["Effretikon"]
+                  address[:city] << "Illnau-Effretikon"
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/app/countries/atlas_engine/ch/address_importer/corrections/open_address/locale_corrector.rb

@@ -0,0 +1,74 @@
+# typed: true
+# frozen_string_literal: true
+
+module AtlasEngine
+  module Ch
+    module AddressImporter
+      module Corrections
+        module OpenAddress
+          class LocaleCorrector
+            class << self
+              extend T::Sig
+
+              sig { params(address: Hash).void }
+              def apply(address)
+                province_code = address[:province_code]
+                zip = address[:zip].to_i
+
+                if [
+                  "ZH",
+                  "BE",
+                  "LU",
+                  "TG",
+                  "SZ",
+                  "AG",
+                  "SO",
+                  "AI",
+                  "AR",
+                  "GL",
+                  "SG",
+                  "SH",
+                  "UR",
+                  "ZG",
+                  "OW",
+                  "NW",
+                  "BL",
+                  "BS",
+                ].to_set.include?(province_code)
+                  address[:locale] = "de"
+                elsif ["GE", "VD", "NE", "JU"].to_set.include?(province_code)
+                  address[:locale] = "fr"
+                elsif province_code == "TI"
+                  address[:locale] = "it"
+                elsif province_code == "FR"
+                  address[:locale] = if zip < 1700 || zip >= 1720 && zip < 1734 || zip >= 1740 && zip < 1791
+                    "fr"
+                  else
+                    "de"
+                  end
+                elsif province_code == "VS"
+                  address[:locale] = if zip < 3900 || zip >= 3960 && zip < 3970 || zip >= 3971 && zip < 3982
+                    "fr"
+                  else
+                    "de"
+                  end
+                elsif province_code == "GR"
+                  if zip < 7000 || zip >= 7013 && zip < 7015 || zip >= 7016 && zip < 7023 ||
+                      zip >= 7031 && zip < 7050 || zip >= 7077 && zip < 7104 || zip >= 7112 && zip < 7122 ||
+                      zip >= 7126 && zip < 7130 || zip >= 7137 && zip < 7147 || zip >= 7151 && zip < 7202 ||
+                      zip >= 7500 && zip < 7562 || zip >= 7602
+                    address[:locale] = "it"
+                  elsif (zip == 7130 || zip >= 7402 && zip < 7404 || zip == 7477) && address[:street].start_with?("Via")
+                    address[:locale] = "it"
+                  else
+                    address[:locale] = "de"
+                  end
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/app/countries/atlas_engine/ch/address_importer/open_address/mapper.rb

@@ -0,0 +1,40 @@
+# typed: true
+# frozen_string_literal: true
+
+module AtlasEngine
+  module Ch
+    module AddressImporter
+      module OpenAddress
+        class Mapper < AtlasEngine::AddressImporter::OpenAddress::DefaultMapper
+          sig do
+            params(feature: AtlasEngine::AddressImporter::OpenAddress::Feature).returns(T::Hash[Symbol, T.untyped])
+          end
+          def map(feature)
+            region, city, street, number, unit, postcode = feature["properties"].values_at(
+              "region",
+              "city",
+              "street",
+              "number",
+              "unit",
+              "postcode",
+            )
+            {
+              source_id: openaddress_source_id(feature),
+              locale: @locale,
+              country_code: @country_code,
+              province_code: region,
+              region1: nil,
+              city: [city],
+              suburb: nil,
+              zip: postcode,
+              street: street,
+              building_and_unit_ranges: housenumber_and_unit(number, unit),
+              latitude: geometry(feature)&.at(1),
+              longitude: geometry(feature)&.at(0),
+            }
+          end
+        end
+      end
+    end
+  end
+end

data/app/countries/atlas_engine/ch/country_profile.yml

@@ -0,0 +1,15 @@
+id: CH
+validation:
+  enabled: true
+  default_matching_strategy: local
+  index_locales:
+    - de
+    - fr
+    - it
+ingestion:
+  open_address:
+    feature_mapper: AtlasEngine::Ch::AddressImporter::OpenAddress::Mapper
+  correctors:
+    open_address:
+      - AtlasEngine::Ch::AddressImporter::Corrections::OpenAddress::LocaleCorrector
+      - AtlasEngine::Ch::AddressImporter::Corrections::OpenAddress::CityCorrector

data/app/countries/atlas_engine/ch/locales/de/country_profile.yml

@@ -0,0 +1,15 @@
+id: CH_DE
+ingestion:
+  data_mapper: AtlasEngine::AddressValidation::Es::DataMappers::DecompoundingDataMapper
+validation:
+  address_parser: AtlasEngine::De::ValidationTranscriber::AddressParser
+  normalized_components:
+  - region2
+  - region3
+  - region4
+  - city_aliases.alias
+  - suburb
+  - street_decompounded
+decompounding_patterns:
+  street:
+  - (?<name>\w+)(?<suffix>allee|gasse|kai|lande|pfad|platz|pl|ring|strasse|str|weg|zeile)(?:\b)

data/app/countries/atlas_engine/ch/locales/de/index_configuration.yml

@@ -0,0 +1,63 @@
+mappings:
+  properties:
+    street:
+      analyzer: street_indexing_analyzer
+      search_analyzer: street_decompounding_analyzer
+    street_stripped:
+      analyzer: street_indexing_analyzer
+      search_analyzer: street_decompounding_analyzer
+    street_decompounded:
+      type: text
+      analyzer: text_analyzer
+      fields:
+        keyword:
+          type: keyword
+settings:
+  index:
+    analysis:
+      analyzer:
+        text_analyzer:
+          filter:
+          - lowercase
+          - german_normalization
+          - icu_folding
+          - strip_special_characters
+        street_indexing_analyzer:
+          tokenizer: standard
+          filter:
+          - lowercase
+          - german_normalization
+          - icu_folding
+          - strip_special_characters
+          - street_suffix_decompounder
+        street_decompounding_analyzer:
+          tokenizer: standard
+          filter:
+          - lowercase
+          - german_normalization
+          - icu_folding
+          - strip_special_characters
+          - street_suffix_decompounder
+          - street_synonyms
+        street_analyzer:
+          filter:
+          - lowercase
+          - german_normalization
+          - icu_folding
+          - strip_special_characters
+          - street_synonyms
+        city_analyzer:
+          filter:
+          - lowercase
+          - german_normalization
+          - icu_folding
+          - strip_special_characters
+          - city_synonyms
+      filter:
+        street_suffix_decompounder:
+          type: pattern_capture
+          preserve_original: "false"
+          patterns:
+            <% country_profile.decompounding_patterns(:street).each do |pattern| %>
+            - <%= pattern %>
+            <% end %>

data/app/countries/atlas_engine/ch/locales/de/synonyms.yml

@@ -0,0 +1,7 @@
+street_synonyms:
+- platz, pl
+- strasse, str
+city_synonyms:
+- in, i
+- sankt, st
+- bei, b

data/app/countries/atlas_engine/ch/locales/fr/synonyms.yml

@@ -0,0 +1,21 @@
+city_synonyms:
+- saint, st
+- sainte, ste
+- saints, sts
+- saintes, stes
+street_synonyms:
+- allée, all
+- avenue, av
+- boulevard, bd
+- centre, ctre
+- chemin, chem
+- impasse, imp
+- passage, pas
+- place, pl
+- rond-point, rpt
+- route, rte
+- saint, st
+- sainte, ste
+- saints, sts
+- saintes, stes
+- sentier, sent

data/app/countries/atlas_engine/cz/country_profile.yml

@@ -0,0 +1,6 @@
+id: CZ
+validation:
+  enabled: true
+  default_matching_strategy: es
+  address_parser: AtlasEngine::Dk::ValidationTranscriber::AddressParser
+  has_provinces: false

data/app/countries/atlas_engine/de/country_profile.yml

@@ -0,0 +1,19 @@
+id: DE
+ingestion:
+  settings:
+    min_zip_edge_ngram: "1"
+    max_zip_edge_ngram: "5"
+  data_mapper: AtlasEngine::AddressValidation::Es::DataMappers::DecompoundingDataMapper
+validation:
+  has_provinces: true
+  address_parser: AtlasEngine::De::ValidationTranscriber::AddressParser
+  normalized_components:
+  - region2
+  - region3
+  - region4
+  - city_aliases.alias
+  - suburb
+  - street_decompounded
+decompounding_patterns:
+  street:
+  - (?<name>\w+)(?<suffix>allee|gasse|kai|lande|pfad|platz|pl|ring|strasse|str|weg|zeile)(?:\b)

data/app/countries/atlas_engine/de/index_configuration.yml

@@ -0,0 +1,64 @@
+mappings:
+  properties:
+    street:
+      analyzer: street_indexing_analyzer
+    street_stripped:
+      analyzer: street_indexing_analyzer
+    street_decompounded:
+      type: text
+      analyzer: text_analyzer
+      fields:
+        keyword:
+          type: keyword
+settings:
+  index:
+    analysis:
+      analyzer:
+        text_analyzer:
+          filter:
+          - lowercase
+          - german_normalization
+          - icu_folding
+          - strip_special_characters
+        keyword_analyzer:
+          filter:
+          - lowercase
+          - german_normalization
+          - icu_folding
+          - strip_special_characters
+        street_indexing_analyzer:
+          tokenizer: standard
+          filter:
+          - lowercase
+          - german_normalization
+          - icu_folding
+          - strip_special_characters
+          - street_suffix_decompounder
+        street_analyzer:
+          filter:
+          - lowercase
+          - german_normalization
+          - icu_folding
+          - strip_special_characters
+          - street_suffix_decompounder
+          - street_synonyms
+        city_analyzer:
+          filter:
+          - lowercase
+          - german_normalization
+          - icu_folding
+          - strip_special_characters
+        edge_ngram_analyzer:
+          filter:
+            - lowercase
+            - german_normalization
+            - icu_folding
+            - strip_special_characters
+      filter:
+        street_suffix_decompounder:
+          type: pattern_capture
+          preserve_original: "false"
+          patterns:
+            <% country_profile.decompounding_patterns(:street).each do |pattern| %>
+            - <%= pattern %>
+            <% end %>

data/app/countries/atlas_engine/de/synonyms.yml

@@ -0,0 +1,2 @@
+street_synonyms:
+- strasse, str

data/app/countries/atlas_engine/de/validation_transcriber/address_parser.rb

@@ -0,0 +1,19 @@
+# typed: true
+# frozen_string_literal: true
+
+module AtlasEngine
+  module De
+    module ValidationTranscriber
+      class AddressParser < AtlasEngine::ValidationTranscriber::AddressParserBase
+        private
+
+        sig { returns(T::Array[Regexp]) }
+        def country_regex_formats
+          @country_regex_formats ||= [
+            /^#{STREET_NO_COMMAS}\s+#{BUILDING_NUM}$/,
+          ]
+        end
+      end
+    end
+  end
+end

data/app/countries/atlas_engine/dk/country_profile.yml

@@ -0,0 +1,6 @@
+id: DK
+validation:
+  enabled: true
+  default_matching_strategy: es
+  has_provinces: false
+  address_parser: AtlasEngine::Dk::ValidationTranscriber::AddressParser

data/app/countries/atlas_engine/dk/synonyms.yml

@@ -0,0 +1,3 @@
+city_synonyms:
+- NV, Nordvest
+- SV, Sydvest

data/app/countries/atlas_engine/dk/validation_transcriber/address_parser.rb

@@ -0,0 +1,21 @@
+# typed: true
+# frozen_string_literal: true
+
+module AtlasEngine
+  module Dk
+    module ValidationTranscriber
+      class AddressParser < AtlasEngine::ValidationTranscriber::AddressParserBase
+        private
+
+        UNIT_NUM = /(?<unit_num>.+)/
+
+        sig { returns(T::Array[Regexp]) }
+        def country_regex_formats
+          @country_regex_formats ||= [
+            /^#{STREET_NO_COMMAS}\s+#{BUILDING_NUM}(,\s*#{UNIT_NUM})?$/,
+          ]
+        end
+      end
+    end
+  end
+end

data/app/countries/atlas_engine/fo/country_profile.yml

@@ -0,0 +1,5 @@
+id: FO
+validation:
+  enabled: true
+  default_matching_strategy: es
+  has_provinces: false

data/app/countries/atlas_engine/fr/address_importer/corrections/open_address/city_corrector.rb

@@ -0,0 +1,28 @@
+# typed: true
+# frozen_string_literal: true
+
+module AtlasEngine
+  module Fr
+    module AddressImporter
+      module Corrections
+        module OpenAddress
+          class CityCorrector
+            class << self
+              extend T::Sig
+
+              sig { params(address: Hash).void }
+              def apply(address)
+                regex = /^(?<city>.*)\s\b(.+)\b\sArrondissement$/
+                m = regex.match(address[:city].first)
+
+                return if m.nil?
+
+                address[:city] = [m["city"]]
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/app/countries/atlas_engine/fr/country_profile.yml

@@ -0,0 +1,13 @@
+id: FR
+ingestion:
+  correctors:
+    open_address:
+      - AtlasEngine::Fr::AddressImporter::Corrections::OpenAddress::CityCorrector
+  settings:
+    number_of_shards: "2"
+    min_zip_edge_ngram: "1"
+    max_zip_edge_ngram: "5"
+validation:
+  address_parser: AtlasEngine::Fr::ValidationTranscriber::AddressParser
+  enabled: true
+  default_matching_strategy: es

data/app/countries/atlas_engine/fr/synonyms.yml

@@ -0,0 +1,21 @@
+city_synonyms:
+- saint, st
+- sainte, ste
+- saints, sts
+- saintes, stes
+street_synonyms:
+- allée, all
+- avenue, av
+- boulevard, bd
+- centre, ctre
+- chemin, chem
+- impasse, imp
+- passage, pas
+- place, pl
+- rond-point, rpt
+- route, rte
+- saint, st
+- sainte, ste
+- saints, sts
+- saintes, stes
+- sentier, sent

data/app/countries/atlas_engine/fr/validation_transcriber/address_parser.rb

@@ -0,0 +1,34 @@
+# typed: true
+# frozen_string_literal: true
+
+module AtlasEngine
+  module Fr
+    module ValidationTranscriber
+      class AddressParser < AtlasEngine::ValidationTranscriber::AddressParserBase
+        private
+
+        sig { override.returns(T::Array[Regexp]) }
+        def country_regex_formats
+          @country_regex_formats ||= [
+            /#{bldg_num_regex}\s+#{street_regex}/i,
+            /#{bldg_num_regex}\s+#{street_regex}(?:,.+)/i,
+          ]
+        end
+
+        sig { returns(Regexp) }
+        def bldg_num_regex
+          # examples:
+          # 100
+          # 100B
+          # 100 bis
+          @bldg_num_regex ||= T.let(/(?<building_num>\d+\s?([a-z]|bis|ter)?)/i, T.nilable(Regexp))
+        end
+
+        sig { returns(Regexp) }
+        def street_regex
+          @street_regex ||= T.let(/(?<street>.+)/, T.nilable(Regexp))
+        end
+      end
+    end
+  end
+end

data/app/countries/atlas_engine/gb/address_validation/es/query_builder.rb

@@ -0,0 +1,98 @@
+# typed: strict
+# frozen_string_literal: true
+
+module AtlasEngine
+  module Gb
+    module AddressValidation
+      module Es
+        class QueryBuilder < AtlasEngine::AddressValidation::Es::QueryBuilder
+          extend T::Sig
+
+          sig { params(address: AtlasEngine::AddressValidation::AbstractAddress, locale: T.nilable(String)).void }
+          def initialize(address, locale = nil)
+            super(address, locale)
+
+            @parsings = T.let(
+              AtlasEngine::Gb::ValidationTranscriber::FullAddressParser
+                .new(address: address).parse,
+              T::Array[ValidationTranscriber::ParsedAddress],
+            )
+          end
+
+          sig { override.returns(T::Hash[String, T.untyped]) }
+          def full_address_query
+            {
+              "query" => {
+                "dis_max" => {
+                  "queries" => @parsings.map { |parsing| query_for_parsing(parsing) },
+                },
+              },
+            }
+          end
+
+          private
+
+          sig { params(parsing: ValidationTranscriber::ParsedAddress).returns(T.nilable(T::Hash[String, T.untyped])) }
+          def locality_clause(parsing)
+            return if parsing.double_dependent_locality.blank? &&
+              parsing.dependent_locality.blank? &&
+              parsing.post_town.blank?
+
+            {
+              "bool" => {
+                "should" => [
+                  {
+                    "match" => {
+                      "region4" => { "query" => parsing.double_dependent_locality || "", "fuzziness" => "auto" },
+                    },
+                  },
+                  {
+                    "match" => {
+                      "region3" => { "query" => parsing.dependent_locality || "", "fuzziness" => "auto" },
+                    },
+                  },
+                  {
+                    "nested" => {
+                      "path" => "city_aliases",
+                      "query" => {
+                        "match" => {
+                          "city_aliases.alias" => { "query" => parsing.post_town || "", "fuzziness" => "auto" },
+                        },
+                      },
+                    },
+                  },
+                ],
+              },
+            }
+          end
+
+          sig { params(parsing: ValidationTranscriber::ParsedAddress).returns(T.nilable(T::Hash[String, T.untyped])) }
+          def query_for_parsing(parsing)
+            {
+              "bool" => {
+                "should" => [
+                  thoroughfare_clause(parsing),
+                  locality_clause(parsing),
+                  zip_clause,
+                ].compact,
+              },
+            }
+          end
+
+          sig { params(parsing: ValidationTranscriber::ParsedAddress).returns(T.nilable(T::Hash[String, T.untyped])) }
+          def thoroughfare_clause(parsing)
+            street_name = [parsing.dependent_street, parsing.street].compact.join(", ")
+
+            return if street_name.blank?
+
+            {
+              "match" => {
+                "street" => { "query" => street_name, "fuzziness" => "auto" },
+              },
+            }
+          end
+        end
+      end
+    end
+  end
+end

data/app/countries/atlas_engine/gb/country_profile.yml

@@ -0,0 +1,10 @@
+id: GB
+validation:
+  has_provinces: true
+  query_builder: AtlasEngine::Gb::AddressValidation::Es::QueryBuilder
+  address_parser: AtlasEngine::ValidationTranscriber::AddressParserNorthAmerica
+ingestion:
+  settings:
+    number_of_shards: "2"
+    min_zip_edge_ngram: "1"
+    max_zip_edge_ngram: "8"