atlas_engine 0.1.0

data/app/lib/atlas_engine/validation_transcriber/address_parser_base.rb

@@ -0,0 +1,201 @@
+# typed: strict
+# frozen_string_literal: true
+
+module AtlasEngine
+  module ValidationTranscriber
+    class AddressParserBase
+      extend T::Sig
+      extend T::Helpers
+      include Formatter
+
+      AddressComponents = T.type_alias { T::Hash[Symbol, String] }
+
+      # Note that parse() returns an array of possible interpretations.
+      # This is because some address lines are ambiguous, and can be interpreted multiple ways.
+      # Example 1:  "123 County Road 45"
+      #  - {building_num: "123", street: "County Road", unit_num: "45"}
+      #  - {building_num: "123", street: "County Road 45"}
+      # Example 2:  "123 E 45"
+      #  - {building_num: "123", street: "E", unit_num: "45"} # 123 E Street Apt 45
+      #  - {building_num: "123", street: "E 45"} # 123 East 45th Street
+
+      # Parts that we slot into address format regular expressions
+
+      BUILDING_NAME = "(?<building_name>[\\w ]+)"
+      BUILDING_NUM =
+        "(?<building_num>("\
+          '([[:digit:]]+\s)?([[:digit:]]+/[[:digit:]]+)|'\
+          '[[:digit:]][[:alpha:][:digit:]/\-]*|'\
+          '[[:alpha:]][[:digit:]][[:alpha:][:digit:]/\-]*'\
+          "))"
+      NUMERIC_ONLY_BUILDING_NUM =
+        "(?<building_num>("\
+          '([[:digit:]]+\s+)?[[:digit:]][[:digit:]/]*[[:digit:]]|'\
+          "[[:digit:]]+"\
+          "))"
+      NON_NUMERIC_STREET = "(?<street>[^[:digit:]/ -].*)"
+      STREET = "(?<street>.+)"
+      STREET_NO_COMMAS = "(?<street>[^,]+)"
+      UNIT_TYPE = '(?<unit_type>[[:alpha:]]+\.?)'
+      UNIT_NUM = '(?<unit_num>[[:alpha:][:digit:]/\-]+)'
+      UNIT_NUM_NO_HYPHEN = "(?<unit_num>[[:alpha:][:digit:]/]+)"
+      PO_BOX = %r{(?:^|\s|/)(?:p(?:ost)?\.?\s*o(?:ffice)?\.?\s*box|box|postal\s*box)\s+(\d+)(?:$|\s)}i
+
+      sig do
+        params(address: AddressValidation::AbstractAddress, preprocessor: T.nilable(AddressParserPreprocessor)).void
+      end
+      def initialize(address:, preprocessor: nil)
+        raise ArgumentError, "country_code cannot be blank in address" if address.country_code.blank?
+
+        @constants = T.let(
+          AtlasEngine::ValidationTranscriber::Constants.instance,
+          AtlasEngine::ValidationTranscriber::Constants,
+        )
+        @country_regex_formats = T.let(nil, T.nilable(T::Array[Regexp]))
+        @address = T.let(address, AddressValidation::TAddress)
+
+        @preprocessor = T.let(
+          preprocessor || AddressParserPreprocessor.new(address: @address),
+          T.nilable(ValidationTranscriber::AddressParserPreprocessor),
+        )
+      end
+
+      sig { returns(T::Array[AddressComponents]) }
+      def parse
+        candidates = []
+
+        address_lines = @preprocessor&.generate_combinations
+
+        return candidates if address_lines&.empty?
+
+        address_lines&.each do |address_line|
+          address_line, po_box = extract_po_box(address_line)
+
+          country_regex_formats.each do |format|
+            m = format.match(address_line)
+            next if m.nil?
+
+            captures = m.named_captures.symbolize_keys
+
+            next if ridiculous?(captures, @address)
+
+            captures = captures.compact_blank.transform_values! do |value|
+              strip_trailing_punctuation(value)
+            end
+            captures[:po_box] = po_box if po_box
+
+            candidates << captures
+          end
+
+          if po_box && candidates.empty?
+            candidates << { po_box: po_box }
+          end
+        end
+
+        candidates.uniq
+      end
+
+      private
+
+      sig { returns(T::Array[Regexp]) }
+      def country_regex_formats
+        []
+      end
+
+      sig { params(address_line: String).returns(T::Array[T.nilable(String)]) }
+      def extract_po_box(address_line)
+        po_box_match = address_line.match(PO_BOX)
+
+        if po_box_match
+          po_box = po_box_match[1]
+          address_line = address_line.gsub(PO_BOX, "").strip
+        else
+          po_box = nil
+        end
+
+        [address_line, po_box]
+      end
+
+      # Return true if something's obviously wrong with this regex match
+      sig do
+        params(
+          captures: T::Hash[Symbol, T.nilable(String)],
+          address: AddressValidation::AbstractAddress,
+        ).returns(T::Boolean)
+      end
+      def ridiculous?(captures, address)
+        building_num = captures[:building_num]&.downcase
+        street = captures[:street]&.downcase
+        unit_num = captures[:unit_num]&.downcase
+        unit_type = captures[:unit_type]&.downcase
+        num_street_space = captures[:num_street_space] # space between building_num and street, if present
+
+        if street.present?
+          return true unless address.address1&.upcase&.include?(street.upcase) ||
+            address.address2&.upcase&.include?(street.upcase)
+        end
+
+        return true if [building_num, street].any? do |token|
+          po_box?(token) || street_suffix?(token)
+        end
+
+        return false if unit_num.present? && secondary_unit_designator?(unit_type)
+
+        return true if [unit_num, unit_type].any? do |token|
+          po_box?(token) || street_suffix?(token)
+        end
+
+        street_tokens_ridiculous?(
+          street: T.must(street),
+          unit_type: unit_type,
+          unit_num: unit_num,
+          num_street_space: num_street_space,
+        )
+      end
+
+      sig { params(token: T.nilable(String)).returns(T::Boolean) }
+      def po_box?(token)
+        return false if token.blank?
+
+        token.match?(/^\s*p\.?\s*o\.?\s*box\s*$/) ||
+          token.match?(/^\s*post\s*office\s*box\s*$/)
+      end
+
+      sig { params(token: T.nilable(String)).returns(T::Boolean) }
+      def secondary_unit_designator?(token)
+        @constants.known?(:secondary_unit_designators, token)
+      end
+
+      sig { params(token: T.nilable(String)).returns(T::Boolean) }
+      def street_suffix?(token)
+        @constants.known?(:street_suffixes, token)
+      end
+
+      sig do
+        params(
+          street: T.nilable(String),
+          unit_type: T.nilable(String),
+          unit_num: T.nilable(String),
+          num_street_space: T.nilable(String),
+        )
+          .returns(T::Boolean)
+      end
+      def street_tokens_ridiculous?(street:, unit_type:, unit_num:, num_street_space:)
+        return false if street.blank?
+
+        street_tokens = street.to_s.split(" ")
+        return true if secondary_unit_designator?(street_tokens[-1]) && !street_suffix?(street_tokens[-1])
+        return true if secondary_unit_designator?(street_tokens[-2]) && !street_suffix?(street_tokens[-1])
+        return true if street_tokens.last&.start_with?("#")
+        return true if unit_type.present? && !secondary_unit_designator?(unit_type)
+
+        ["st", "nd", "rd", "th", "er", "eme"].each do |suffix|
+          return true if unit_num&.end_with?(suffix)
+          return true if num_street_space.nil? && street.split(" ").first == suffix
+        end
+
+        false
+      end
+    end
+  end
+end

data/app/lib/atlas_engine/validation_transcriber/address_parser_factory.rb

@@ -0,0 +1,27 @@
+# typed: strict
+# frozen_string_literal: true
+
+module AtlasEngine
+  module ValidationTranscriber
+    class AddressParserFactory
+      class << self
+        extend T::Sig
+
+        sig do
+          params(address: AddressValidation::AbstractAddress, locale: T.nilable(String)).returns(AddressParserBase)
+        end
+        def create(address:, locale: nil)
+          raise ArgumentError, "country_code cannot be nil" if address.country_code.nil?
+
+          profile = CountryProfile.for(T.must(address.country_code), locale)
+
+          if locale.nil? && profile.validation.multi_locale?
+            raise ArgumentError, "#{address.country_code} is a multi-locale country and requires a locale"
+          end
+
+          profile.validation.address_parser.new(address: address)
+        end
+      end
+    end
+  end
+end

data/app/lib/atlas_engine/validation_transcriber/address_parser_north_america.rb

@@ -0,0 +1,39 @@
+# typed: strict
+# frozen_string_literal: true
+
+module AtlasEngine
+  module ValidationTranscriber
+    class AddressParserNorthAmerica < AddressParserBase
+      private
+
+      sig { returns(T::Array[Regexp]) }
+      def country_regex_formats
+        @country_regex_formats ||= [
+          "#{BUILDING_NUM}(?<num_street_space>\s+)#{STREET}",
+          "#{NUMERIC_ONLY_BUILDING_NUM}#{NON_NUMERIC_STREET}",
+        ].map do |building_and_street_expr|
+          north_american_variants(building_and_street_expr)
+        end.flatten.uniq
+      end
+
+      # {building_num} {street}
+      # {unit_num}-{building_num} {street}
+      # {building_num} {street} {unit_type} {unit_num}
+      # {building_num} {street} #{unit_num}
+      # {building_num} {street} {unit_num}
+      # {building_num} {street} - {unit_num}
+      sig { params(building_and_street_expr: String).returns(T::Array[Regexp]) }
+      def north_american_variants(building_and_street_expr)
+        [
+          /^#{building_and_street_expr}$/,
+          /^(#{UNIT_NUM_NO_HYPHEN}-)?#{building_and_street_expr}$/,
+          /^#{building_and_street_expr}\s+#{UNIT_TYPE}\s+#{UNIT_NUM}/,
+          /^#{building_and_street_expr}\s+\#\s*#{UNIT_NUM}/,
+          /^#{building_and_street_expr}\s+-\s+#{UNIT_NUM}/,
+          /^#{BUILDING_NAME}\s#{building_and_street_expr}$/,
+          /^#{UNIT_TYPE}\s+#{UNIT_NUM_NO_HYPHEN}\s+#{building_and_street_expr}$/,
+        ]
+      end
+    end
+  end
+end

data/app/lib/atlas_engine/validation_transcriber/address_parser_oceanic.rb

@@ -0,0 +1,17 @@
+# typed: strict
+# frozen_string_literal: true
+
+module AtlasEngine
+  module ValidationTranscriber
+    class AddressParserOceanic < AddressParserBase
+      private
+
+      sig { returns(T::Array[Regexp]) }
+      def country_regex_formats
+        @country_regex_formats ||= [
+          %r{^((?<unit_num>[[:alpha:]0-9]+)/)?(?<building_num>[0-9][[:alpha:]0-9]*)\s+(?<street>.+)$},
+        ]
+      end
+    end
+  end
+end

data/app/lib/atlas_engine/validation_transcriber/address_parser_preprocessor.rb

@@ -0,0 +1,132 @@
+# typed: strict
+# frozen_string_literal: true
+
+module AtlasEngine
+  module ValidationTranscriber
+    class AddressParserPreprocessor
+      include Formatter
+      include AddressParsingHelper
+      extend T::Sig
+
+      sig do
+        params(
+          address: AddressValidation::TAddress,
+        ).void
+      end
+      def initialize(address:)
+        raise ArgumentError, "country_code cannot be blank in address" if address.country_code.blank?
+
+        @country = T.let(Worldwide.region(code: address.country_code), Worldwide::Region)
+        @address = address
+        @combinations = T.let(Set.new, T::Set[T.nilable(String)])
+      end
+
+      sig { returns(T::Array[String]) }
+      def generate_combinations
+        @combinations << @address.address1
+        @combinations << @address.address2
+        @combinations << combined_address_lines
+        @combinations << address_1_stripped_of_known_components_excluding_zip
+        @combinations << address_1_stripped_of_known_components
+        @combinations << address_1_sliced_on_street
+
+        @combinations.compact_blank.uniq
+      end
+
+      private
+
+      sig { returns(String) }
+      def combined_address_lines
+        [@address.address1, @address.address2].compact_blank.join(" ")
+      end
+
+      sig { returns(T.nilable(String)) }
+      def address_1_stripped_of_known_components_excluding_zip
+        @address_1_stripped_of_known_components_excluding_zip ||= T.let(
+          begin
+            # rubocop:disable Lint/NoReturnInBeginEndBlocks
+            return if @address.address1.blank? || @address.nil?
+            # rubocop:enable Lint/NoReturnInBeginEndBlocks
+
+            components_to_strip = [
+              @address.address2,
+              @address.city,
+              possible_province_words,
+              possible_country_words,
+            ]
+
+            address_line = T.must(@address.address1)
+            components_to_strip.flatten.compact_blank.each do |address_component|
+              address_line = strip_word(address_line, address_component)
+            end
+
+            address_line
+          end,
+          T.nilable(String),
+        )
+      end
+
+      sig { returns(T.nilable(String)) }
+      def address_1_stripped_of_known_components
+        modified_address1 = address_1_stripped_of_known_components_excluding_zip
+        possible_zip = possible_zip_word
+        return if modified_address1.blank? || possible_zip.blank?
+
+        strip_word(modified_address1, possible_zip)
+      end
+
+      sig { returns(T.nilable(String)) }
+      def address_1_sliced_on_street
+        return unless @country.legacy_code == "US" && @address.address1.present?
+
+        address_line_tokens = T.must(@address.address1).split(" ")
+        street_suffix_index = address_line_tokens.rindex do |token|
+          street_suffix?(token)
+        end
+
+        return unless street_suffix_index
+
+        index_to_slice = if directional?(address_line_tokens[street_suffix_index + 1])
+          street_suffix_index + 1
+        else
+          street_suffix_index
+        end
+
+        slice_at_index(address_line_tokens, index_to_slice)
+      end
+
+      sig { returns(T::Array[String]) }
+      def possible_province_words
+        province_code = @address.province_code
+        return [] if province_code.blank?
+
+        zone = @country.zone(code: province_code)
+        return [] unless zone.province?
+
+        [
+          province_code,
+          zone.code_alternates,
+          zone.name_alternates,
+          zone.full_name,
+        ].flatten.compact
+      end
+
+      sig { returns(T::Array[String]) }
+      def possible_country_words
+        [@country.legacy_code.to_s, @country.full_name, @country.name_alternates].flatten.compact
+      end
+
+      sig { returns(T.nilable(String)) }
+      def possible_zip_word
+        return if @address.zip.blank?
+
+        @address.zip if @country.valid_zip?(@address.zip)
+      end
+
+      sig { params(tokens: T::Array[String], index: Integer).returns(String) }
+      def slice_at_index(tokens, index)
+        T.must(tokens[..index]).join(" ")
+      end
+    end
+  end
+end

data/app/lib/atlas_engine/validation_transcriber/address_parsing_helper.rb

@@ -0,0 +1,38 @@
+# typed: true
+# frozen_string_literal: true
+
+module AtlasEngine
+  module ValidationTranscriber
+    module AddressParsingHelper
+      extend T::Sig
+
+      sig { returns(Constants) }
+      def address_constants
+        @address_constants ||= T.let(
+          Constants.instance,
+          T.nilable(Constants),
+        )
+      end
+
+      sig { params(token: T.nilable(String)).returns(T::Boolean) }
+      def directional?(token)
+        return false if token.blank?
+
+        downcased = token.downcase
+        english = address_constants.translations_fr_en[downcased.to_sym] || downcased
+
+        address_constants.known?(:directionals, english)
+      end
+
+      sig { params(token: T.nilable(String)).returns(T::Boolean) }
+      def street_suffix?(token)
+        return false if token.blank?
+
+        downcased = token.downcase
+        english = address_constants.translations_fr_en[downcased.to_sym] || downcased
+
+        address_constants.known?(:street_suffixes, english)
+      end
+    end
+  end
+end

data/app/lib/atlas_engine/validation_transcriber/address_parsings.rb

@@ -0,0 +1,54 @@
+# typed: strict
+# frozen_string_literal: true
+
+module AtlasEngine
+  module ValidationTranscriber
+    class AddressParsings
+      extend T::Sig
+      include LogHelper
+
+      ParsedComponents = T.type_alias { T::Hash[Symbol, String] }
+
+      sig { returns(T::Array[ParsedComponents]) }
+      attr_reader :parsings
+
+      sig { params(address_input: AddressValidation::AbstractAddress, locale: T.nilable(String)).void }
+      def initialize(address_input:, locale: nil)
+        @parsings = T.let(
+          begin
+            if address_input.country_code.blank?
+              []
+            else
+              parsing_result = AddressParserFactory.create(address: address_input, locale: locale).parse
+              log_unparsable_address(address_input) if parsing_result.empty?
+              parsing_result
+            end
+          end,
+          T::Array[ParsedComponents],
+        )
+      end
+
+      sig { returns(T::Boolean) }
+      def describes_po_box?
+        parsings.any? { |parsing| parsing[:po_box] }
+      end
+
+      sig { returns(T::Array[String]) }
+      def potential_streets
+        potential_parsings = parsings.pluck(:street).compact
+        potential_parsings << "po box" if describes_po_box?
+        potential_parsings.uniq
+      end
+
+      sig { returns(T::Array[String]) }
+      def potential_building_numbers
+        parsings.pluck(:building_num).compact.uniq
+      end
+
+      sig { params(address_input: AddressValidation::AbstractAddress).void }
+      def log_unparsable_address(address_input)
+        log_info("[AddressValidation] Unable to parse address lines", address_input.to_h.except(:phone))
+      end
+    end
+  end
+end

data/app/lib/atlas_engine/validation_transcriber/constants.rb

@@ -0,0 +1,50 @@
+# typed: true
+# frozen_string_literal: true
+
+require "singleton"
+
+module AtlasEngine
+  module ValidationTranscriber
+    class Constants
+      extend T::Sig
+      include Singleton
+
+      TRANSCRIBER_FILE = File.join(AtlasEngine::Engine.root, "db/data/transcriber.yml").freeze
+
+      class << self
+        def instance
+          @instance ||= new
+        end
+
+        def create_accessor_methods(yaml_hash)
+          yaml_hash.each do |constant_name, _|
+            define_method(constant_name.to_s) do
+              @data[constant_name]
+            end
+          end
+        end
+      end
+
+      sig { void }
+      def initialize
+        @data ||= load_yaml_file(TRANSCRIBER_FILE)
+        self.class.create_accessor_methods(@data)
+      end
+
+      sig { params(constant_type: Symbol, value: T.nilable(String)).returns(T::Boolean) }
+      def known?(constant_type, value)
+        constants = @data[constant_type]
+        return false if constants.blank? || value.blank?
+
+        downcased = value.delete_suffix(".").downcase
+        constants.key?(downcased.to_sym) || constants.value?(downcased)
+      end
+
+      private
+
+      def load_yaml_file(filename)
+        YAML.load_file(filename, freeze: true).deep_symbolize_keys
+      end
+    end
+  end
+end

data/app/lib/atlas_engine/validation_transcriber/english_street_parser.rb

@@ -0,0 +1,59 @@
+# typed: false
+# frozen_string_literal: true
+
+module AtlasEngine
+  module ValidationTranscriber
+    class EnglishStreetParser
+      include AddressParsingHelper
+
+      def initialize
+        super
+      end
+
+      def parse(street:)
+        return {} if street.blank?
+
+        # Expected format: [pre_directional, name, suffix, post_directional]
+        # Note that pre_directional and post_directional may be absent, one word ("East"), or two words ("North East")
+
+        pre_directional = nil
+        suffix = nil
+        post_directional = nil
+
+        tokens = street.split(" ")
+
+        if directional?(tokens[0])
+          if directional?(tokens[1])
+            pre_directional = tokens[0..1].join(" ")
+            tokens = tokens[2..-1]
+          else
+            pre_directional = tokens[0]
+            tokens = tokens[1..-1]
+          end
+        end
+
+        if directional?(tokens[-1])
+          if directional?(tokens[-2])
+            post_directional = tokens[-2..-1].join(" ")
+            tokens = tokens[0..-3]
+          else
+            post_directional = tokens[-1]
+            tokens = tokens[0..-2]
+          end
+        end
+
+        if street_suffix?(tokens[-1])
+          suffix = tokens[-1]
+          tokens = tokens[0..-2]
+        end
+
+        {
+          pre_directional: pre_directional,
+          name: tokens.join(" "),
+          suffix: suffix,
+          post_directional: post_directional,
+        }.compact_blank.to_h
+      end
+    end
+  end
+end

data/app/lib/atlas_engine/validation_transcriber/formatter.rb

@@ -0,0 +1,46 @@
+# typed: strict
+# frozen_string_literal: true
+
+module AtlasEngine
+  module ValidationTranscriber
+    module Formatter
+      extend T::Sig
+
+      sig { params(text: T.nilable(String)).returns(T.nilable(String)) }
+      def strip_trailing_punctuation(text)
+        text.presence&.sub(/[\s,\-]+$/, "")
+      end
+
+      sig { params(haystack: String, needle: String).returns(String) }
+      def strip_word(haystack, needle)
+        string = haystack.sub(/[\s](#{Regexp.escape(needle)})([\s]|$)/i, " ").strip
+        string = string.sub(/[\s,](#{Regexp.escape(needle)})([\s,]|$)/i, "").strip
+        string = strip_trailing_punctuation(string)
+        string || ""
+      end
+
+      sig do
+        params(
+          address1: String,
+          address2: String,
+          city: String,
+          province_code: String,
+          zip: String,
+          country_code: String,
+          phone: String,
+        ).returns(AddressValidation::Address)
+      end
+      def build_address(address1: "", address2: "", city: "", province_code: "", zip: "", country_code: "", phone: "")
+        AddressValidation::Address.new(
+          address1: address1,
+          address2: address2,
+          city: city,
+          province_code: province_code,
+          zip: zip,
+          country_code: country_code,
+          phone: phone,
+        )
+      end
+    end
+  end
+end