atlas_engine 0.1.0

data/app/countries/atlas_engine/li/synonyms.yml

@@ -0,0 +1,6 @@
+street_synonyms:
+- platz, pl
+- strasse, str
+city_synonyms:
+- in, i
+- sankt, st

data/app/countries/atlas_engine/lt/country_profile.yml

@@ -0,0 +1,6 @@
+id: LT
+validation:
+  enabled: true
+  default_matching_strategy: es
+  address_parser: AtlasEngine::Lt::ValidationTranscriber::AddressParser
+  has_provinces: false

data/app/countries/atlas_engine/lt/synonyms.yml

@@ -0,0 +1,7 @@
+street_synonyms:
+- gatvė, gvė, g
+- alėja, al
+- aikštė, a
+- skersgatvis, skg
+- plentas, pl
+- prospektas, pr

data/app/countries/atlas_engine/lt/validation_transcriber/address_parser.rb

@@ -0,0 +1,24 @@
+# typed: true
+# frozen_string_literal: true
+
+module AtlasEngine
+  module Lt
+    module ValidationTranscriber
+      class AddressParser < AtlasEngine::ValidationTranscriber::AddressParserBase
+        private
+
+        STREET = "(?<street>.+?)"
+        BUILDING_NUM = "(?<building_num>[0-9]+[[:alpha:]]*)"
+        UNIT_NUM = "(?<unit_num>[0-9]+[[:alpha:]]*)"
+
+        sig { override.returns(T::Array[Regexp]) }
+        def country_regex_formats
+          @country_regex_formats ||= [
+            /^#{STREET}\s#{BUILDING_NUM}/i,
+            /^#{STREET}\s#{BUILDING_NUM}-#{UNIT_NUM}/i,
+          ]
+        end
+      end
+    end
+  end
+end

data/app/countries/atlas_engine/lu/address_importer/corrections/open_address/locale_corrector.rb

@@ -0,0 +1,54 @@
+# typed: true
+# frozen_string_literal: true
+
+module AtlasEngine
+  module Lu
+    module AddressImporter
+      module Corrections
+        module OpenAddress
+          class LocaleCorrector
+            class << self
+              extend T::Sig
+
+              FRENCH_STREET_PREFIXES = [
+                "Allée",
+                "Avenue",
+                "Boulevard",
+                "Centre",
+                "Ceinture",
+                "Chemin",
+                "Cité",
+                "Domaine",
+                "Impasse",
+                "Maison",
+                "Montée",
+                "Parc",
+                "Passage",
+                "Place",
+                "Plateau",
+                "Porte",
+                "Rond-Point",
+                "Rond Point",
+                "Route",
+                "Rue",
+                "Sentier",
+                "Zone",
+              ]
+
+              sig { params(address: Hash).void }
+              def apply(address)
+                street = address[:street]
+
+                address[:locale] = if FRENCH_STREET_PREFIXES.any? { |prefix| street.start_with?(prefix) }
+                  "fr"
+                else
+                  "lb"
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/app/countries/atlas_engine/lu/country_profile.yml

@@ -0,0 +1,12 @@
+id: LU
+validation:
+  enabled: true
+  has_provinces: true
+  default_matching_strategy: es
+  index_locales:
+    - fr
+    - lb
+ingestion:
+  correctors:
+    open_address:
+      - AtlasEngine::Lu::AddressImporter::Corrections::OpenAddress::LocaleCorrector

data/app/countries/atlas_engine/nl/address_importer/corrections/open_address/city_corrector.rb

@@ -0,0 +1,25 @@
+# typed: true
+# frozen_string_literal: true
+
+module AtlasEngine
+  module Nl
+    module AddressImporter
+      module Corrections
+        module OpenAddress
+          class CityCorrector
+            class << self
+              extend T::Sig
+
+              sig { params(address: Hash).void }
+              def apply(address)
+                if address[:city] == ["'s-Gravenhage"]
+                  address[:city] << "Den Haag"
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/app/countries/atlas_engine/nl/country_profile.yml

@@ -0,0 +1,18 @@
+id: NL
+ingestion:
+  correctors:
+    open_address:
+      - AtlasEngine::Nl::AddressImporter::Corrections::OpenAddress::CityCorrector
+  data_mapper: AtlasEngine::AddressValidation::Es::DataMappers::DecompoundingDataMapper
+validation:
+  enabled: true
+  has_provinces: true
+  default_matching_strategy: es
+  normalized_components:
+  - street_decompounded
+  address_parser: AtlasEngine::Nl::ValidationTranscriber::AddressParser
+decompounding_patterns:
+  street:
+  - (?<name>\w+)(?<suffix>dwarsstraat|dwstr|dwarsweg|dwwg|dijk|dk|gracht|gr|kade|kd|kanaal|kan)(?:\b)
+  - (?<name>\w+)(?<suffix>laan|leane|loane|ln|park|pk|plantsoen|plnts|plein|pln|singel|sngl)(?:\b)
+  - (?<name>\w+)(?<suffix>straat|strjitte|str|straatweg|strwg|weg|wg)(?:\b)

data/app/countries/atlas_engine/nl/index_configuration.yml

@@ -0,0 +1,52 @@
+mappings:
+  properties:
+    street:
+      analyzer: street_indexing_analyzer
+      search_analyzer: street_decompounding_analyzer
+    street_stripped:
+      analyzer: street_indexing_analyzer
+      search_analyzer: street_decompounding_analyzer
+    street_decompounded:
+      type: text
+      analyzer: text_analyzer
+      fields:
+        keyword:
+          type: keyword
+settings:
+  index:
+    analysis:
+      analyzer:
+        text_analyzer:
+          filter:
+          - lowercase
+          - icu_folding
+          - strip_special_characters
+        street_indexing_analyzer:
+          tokenizer: standard
+          filter:
+          - lowercase
+          - icu_folding
+          - strip_special_characters
+          - street_suffix_decompounder
+        street_decompounding_analyzer:
+          tokenizer: standard
+          filter:
+          - lowercase
+          - icu_folding
+          - strip_special_characters
+          - street_suffix_decompounder
+          - street_synonyms
+        street_analyzer:
+          filter:
+          - lowercase
+          - icu_folding
+          - strip_special_characters
+          - street_synonyms
+      filter:
+        street_suffix_decompounder:
+          type: pattern_capture
+          preserve_original: "false"
+          patterns:
+            <% country_profile.decompounding_patterns(:street).each do |pattern| %>
+            - <%= pattern %>
+            <% end %>

data/app/countries/atlas_engine/nl/synonyms.yml

@@ -0,0 +1,92 @@
+street_synonyms:
+# dutch
+## directionals
+- noord, n
+- oost, o
+- west, w
+- zuid, z
+## street suffixes
+- boulevard, bd, bld, boulev
+- dijk, dk
+- dwarsstraat, dwstr
+- dwarsweg, dwwg
+- gracht, gr
+- kade, kd
+- kanaal, kan
+- laan, leane, loane, ln
+- park, pk
+- plantsoen, plnts
+- plein, pln, ln
+- singel, sngl
+- square, sq
+- steenweg, stwg
+- straat, str
+- straatweg, strwg
+- van, v
+- weg, wg
+## titles
+- aalmoezenier, aalm  # Chaplain
+- admiraal, adm  # Admiral
+- baron, bar  # Baron
+- baronesse, bsse  # Baroness
+- bisschop, biss  # Bishop
+- burgemeester, burg  # Mayor
+- commissaris, comm  # Commissioner
+- deken, dkn  # Deacon
+- dokter, doctor, dr  # Doctor
+- dominee, ds  # Minister, Vicar, Padre
+- douairière, dre  # Dowager
+- gebroeders, gebr  # Brothers
+- generaal, gen  # General
+- graaf, gr  # Duke
+- gravin, gr  # Duchess
+- hertog, htg  # Earl
+- ingenieur, ir  # Engineer
+- jonkheer, jhr  # Esquire
+- kanunnik, kan  # Canon
+- kapelaan, kap  # Curate, Chaplain
+- kapitein, kapt  # Captain
+- kardinaal, kard  # Cardinal
+- keizer, kzr  # Emperor
+- kolonel, kol  # Colonel
+- koning, kon  # King
+- koningin, kon  # Queen
+- luitenant generaal, lt gen  # Lieutenant-General
+- majoor, maj  # Major
+- meester, mr  # Mister (academic grade)
+- minister, min  # Minister (political)
+- monseigneur, mgr  # Monseigneur
+- notaris, not  # Notary
+- pastoor, past  # Pastor
+- pater, ptr  # Father (religious)
+- paters, ptrs  # Fathers (religious)
+- president, pres  # President
+- prins, pr  # Prince
+- princes, pr  # Princess
+- professor, prof  # Professor
+- rector, rect  # Rector
+- ridder, rdr  # Knight
+- secretaris, secr  # Secretary
+- sinjeur, sinj  # Cleric
+- sint, st  # Saint
+- wethouder, weth  # Alderman
+- zuster, zr  # Sister (religious)
+- zusters, zrs  # Sisters (religious)
+## common words
+- aan de, a d, ad
+- het, 't
+- van, v
+# friesian
+## directionals
+- east, o
+- noard, n
+- sud, s
+## street suffixes
+- strjitte, str
+
+city_synonyms:
+## common words
+- aan de, a d, ad
+- het, 't
+- sint, st
+- van, v

data/app/countries/atlas_engine/nl/validation_transcriber/address_parser.rb

@@ -0,0 +1,85 @@
+# typed: true
+# frozen_string_literal: true
+
+module AtlasEngine
+  module Nl
+    module ValidationTranscriber
+      class AddressParser < AtlasEngine::ValidationTranscriber::AddressParserBase
+        private
+
+        STREET = "(?<street>.+)"
+        NUMBERED_STREET = "(?<street>.+\s+[0-9]+)"
+        BUILDING_NUM = "(?<building_num>[0-9]+[:alpha:]*)"
+        UNIT_NUM = "(?<unit_num>[[:alnum:]]+)"
+        PO_BOX = /\b(?<box_type>pb|postbus|antwoordnummer)\s+(?<number>\d+)\b/i
+        # since not all street synonyms are street suffixes, we cannot read them from the synonyms file
+        # TODO synonyms in the file should be grouped by type
+        STREET_SUFFIXES = %r{
+          \A(
+            dwarsstraat|dwstr|dwarsweg|dwwg|dijk|dk|gracht|gr|kade|kd|kanaal|kan
+            |laan|leane|loane|ln|park|pk|plantsoen|plnts|plein|pln|singel|sngl
+            |straat|strjitte|str|straatweg|strwg|weg|wg
+          )\z
+        }ix
+
+        sig { returns(T::Array[Regexp]) }
+        def country_regex_formats
+          @country_regex_formats ||= [
+            /^#{STREET},?\s+#{BUILDING_NUM}$/,
+            /^#{NUMBERED_STREET},?\s+#{BUILDING_NUM}$/,
+            /^#{STREET},?\s+#{BUILDING_NUM}[\s,-]+#{UNIT_NUM}$/,
+            /^#{NUMBERED_STREET},?\s+#{BUILDING_NUM}[\s,-]+#{UNIT_NUM}$/,
+          ]
+        end
+
+        sig { override.params(address_line: String).returns(T::Array[T.nilable(String)]) }
+        def extract_po_box(address_line)
+          po_box_match = address_line.match(PO_BOX)
+
+          if po_box_match
+            po_box = po_box_match["number"]
+            address_line = address_line.gsub(PO_BOX, "").strip.delete_suffix(",")
+          else
+            po_box = nil
+          end
+
+          [address_line, po_box]
+        end
+
+        # Return true if something's obviously wrong with this regex match
+        sig do
+          override.params(
+            captures: T::Hash[Symbol, T.nilable(String)],
+            address: AddressValidation::TAddress,
+          ).returns(T::Boolean)
+        end
+        def ridiculous?(captures, address)
+          building_num = captures[:building_num]&.downcase
+          street = captures[:street]&.downcase
+          unit_num = captures[:unit_num]&.downcase
+
+          if street.present?
+            return true unless address.address1&.upcase&.include?(street.upcase) ||
+              address.address2&.upcase&.include?(street.upcase)
+          end
+
+          [building_num, unit_num].any? do |token|
+            po_box?(token) || street_suffix?(token)
+          end
+        end
+
+        sig { override.params(token: T.nilable(String)).returns(T::Boolean) }
+        def po_box?(token)
+          return false if token.blank?
+
+          token.match?(PO_BOX)
+        end
+
+        sig { override.params(token: T.nilable(String)).returns(T::Boolean) }
+        def street_suffix?(token)
+          token.present? && token.match?(STREET_SUFFIXES)
+        end
+      end
+    end
+  end
+end

data/app/countries/atlas_engine/no/country_profile.yml

@@ -0,0 +1,5 @@
+id: 'NO' # YAML treats NO as a boolean, so we need to quote it
+ingestion:
+  settings:
+    min_zip_edge_ngram: "1"
+    max_zip_edge_ngram: "4"

data/app/countries/atlas_engine/nz/country_profile.yml

@@ -0,0 +1,3 @@
+id: NZ
+validation:
+  address_parser: AtlasEngine::ValidationTranscriber::AddressParserOceanic

data/app/countries/atlas_engine/pl/country_profile.yml

@@ -0,0 +1,5 @@
+id: PL
+validation:
+  address_parser: AtlasEngine::Pl::ValidationTranscriber::AddressParser
+  enabled: true
+  default_matching_strategy: es

data/app/countries/atlas_engine/pl/validation_transcriber/address_parser.rb

@@ -0,0 +1,19 @@
+# typed: true
+# frozen_string_literal: true
+
+module AtlasEngine
+  module Pl
+    module ValidationTranscriber
+      class AddressParser < AtlasEngine::ValidationTranscriber::AddressParserBase
+        private
+
+        sig { returns(T::Array[Regexp]) }
+        def country_regex_formats
+          @country_regex_formats ||= [
+            %r{^(?<street>.+)\s+(?<building_num>[0-9][[:alpha:]0-9]*)(\s*/\s*(?<unit_num>[[:alpha:]0-9]+))?$},
+          ]
+        end
+      end
+    end
+  end
+end

data/app/countries/atlas_engine/pt/address_importer/corrections/open_address/city_corrector.rb

@@ -0,0 +1,32 @@
+# typed: true
+# frozen_string_literal: true
+
+module AtlasEngine
+  module Pt
+    module AddressImporter
+      module Corrections
+        module OpenAddress
+          class CityCorrector
+            class << self
+              extend T::Sig
+
+              sig { params(address: Hash).void }
+              def apply(address)
+                city_aliases = [
+                  { city_name: "Vila Nova De Gaia", alias: "Gaia" },
+                  { city_name: "Portela Lrs", alias: "Portela" },
+                  { city_name: "Vila Chã Vcd", alias: "Vila Chã" },
+                  { city_name: "Alverca Do Ribatejo", alias: "Alverca" },
+                ]
+
+                city_aliases.each do |city_alias|
+                  address[:city] << city_alias[:alias] if address[:city].include?(city_alias[:city_name])
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/app/countries/atlas_engine/pt/address_importer/open_address/mapper.rb

@@ -0,0 +1,39 @@
+# typed: true
+# frozen_string_literal: true
+
+module AtlasEngine
+  module Pt
+    module AddressImporter
+      module OpenAddress
+        class Mapper < AtlasEngine::AddressImporter::OpenAddress::DefaultMapper
+          sig do
+            params(feature: AtlasEngine::AddressImporter::OpenAddress::Feature).returns(T::Hash[Symbol, T.untyped])
+          end
+          def map(feature)
+            city, street, number, unit, postcode = feature["properties"].values_at(
+              "city",
+              "street",
+              "number",
+              "unit",
+              "postcode",
+            )
+            {
+              source_id: openaddress_source_id(feature),
+              locale: @locale,
+              country_code: "PT",
+              province_code: province_code_from_zip(postcode),
+              # Omitted: region1..4
+              city: [city.titleize],
+              suburb: nil,
+              zip: postcode,
+              street: street.titleize,
+              building_and_unit_ranges: housenumber_and_unit(number, unit),
+              latitude: geometry(feature)&.at(1),
+              longitude: geometry(feature)&.at(0),
+            }
+          end
+        end
+      end
+    end
+  end
+end

data/app/countries/atlas_engine/pt/country_profile.yml

@@ -0,0 +1,10 @@
+id: PT
+ingestion:
+  correctors:
+    open_address:
+      - AtlasEngine::Pt::AddressImporter::Corrections::OpenAddress::CityCorrector
+  open_address:
+    feature_mapper: AtlasEngine::Pt::AddressImporter::OpenAddress::Mapper
+validation:
+  enabled: true
+  default_matching_strategy: es

data/app/countries/atlas_engine/pt/synonyms.yml

@@ -0,0 +1,7 @@
+city_synonyms:
+- GDM, Gondomar
+- GMR, Guimarães
+- VCD, Vila do Conde
+- VGS, Vagos
+- VN, Vila Nova De
+- VN, Vila Nova Da

data/app/countries/atlas_engine/sa/country_profile.yml

@@ -0,0 +1,10 @@
+id: SA
+validation:
+  enabled: false
+  default_matching_strategy: es
+  has_provinces: false
+  restrictions:
+  restrictions:
+    - class: AtlasEngine::Restrictions::UnsupportedScript
+      params:
+        supported_script: :Arabic

data/app/countries/atlas_engine/se/country_profile.yml

@@ -0,0 +1,5 @@
+id: SE
+ingestion:
+  settings:
+    min_zip_edge_ngram: "1"
+    max_zip_edge_ngram: "6"

data/app/countries/atlas_engine/tt/address_importer/open_address/mapper.rb

@@ -0,0 +1,38 @@
+# typed: true
+# frozen_string_literal: true
+
+module AtlasEngine
+  module Tt
+    module AddressImporter
+      module OpenAddress
+        class Mapper < AtlasEngine::AddressImporter::OpenAddress::DefaultMapper
+          sig do
+            params(feature: AtlasEngine::AddressImporter::OpenAddress::Feature).returns(T::Hash[Symbol, T.untyped])
+          end
+          def map(feature)
+            city, street, number, unit, postcode = feature["properties"].values_at(
+              "city",
+              "street",
+              "number",
+              "unit",
+              "postcode",
+            )
+            {
+              source_id: openaddress_source_id(feature),
+              locale: @locale,
+              country_code: "TT",
+              province_code: nil,
+              city: ["Chaguanas"],
+              suburb: city,
+              zip: postcode,
+              street: street,
+              building_and_unit_ranges: housenumber_and_unit(number, unit),
+              latitude: geometry(feature)&.at(1),
+              longitude: geometry(feature)&.at(0),
+            }
+          end
+        end
+      end
+    end
+  end
+end

data/app/countries/atlas_engine/tt/country_profile.yml

@@ -0,0 +1,7 @@
+id: TT
+ingestion:
+  open_address:
+    feature_mapper: AtlasEngine::Tt::AddressImporter::OpenAddress::Mapper
+validation:
+  address_parser: AtlasEngine::ValidationTranscriber::AddressParserNorthAmerica
+  has_provinces: false

data/app/countries/atlas_engine/us/country_profile.yml

@@ -0,0 +1,12 @@
+id: US
+validation:
+  enabled: true
+  default_matching_strategy: es_street
+  city_fields:
+    - city_aliases
+  address_parser: AtlasEngine::ValidationTranscriber::AddressParserNorthAmerica
+ingestion:
+  settings:
+    number_of_shards: "7"
+    min_zip_edge_ngram: "1"
+    max_zip_edge_ngram: "10"