atlas_engine 0.1.0

data/app/models/atlas_engine/address_validation/token/comparator.rb

@@ -0,0 +1,44 @@
+# typed: true
+# frozen_string_literal: true
+
+require "rubygems/text"
+
+module AtlasEngine
+  module AddressValidation
+    class Token
+      class Comparator
+        extend T::Sig
+        include Gem::Text
+
+        sig { returns(Token) }
+        attr_reader :left, :right
+
+        sig { params(left_token: Token, right_token: Token).void }
+        def initialize(left_token, right_token)
+          @left = T.let(left_token, Token)
+          @right = T.let(right_token, Token)
+        end
+
+        sig { returns(Comparison) }
+        def compare
+          left_value = left.value
+          right_value = right.value
+
+          if left_value == right_value
+            Comparison.new(left: left, right: right, qualifier: :equal, edit_distance: 0)
+          else
+            edit = levenshtein_distance(left_value, right_value)
+
+            if right_value.start_with?(left_value) || left_value.start_with?(right_value)
+              Comparison.new(left: left, right: right, qualifier: :prefix, edit_distance: edit)
+            elsif right_value.end_with?(left_value) || left_value.end_with?(right_value)
+              Comparison.new(left: left, right: right, qualifier: :suffix, edit_distance: edit)
+            else
+              Comparison.new(left: left, right: right, qualifier: :comp, edit_distance: edit)
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/app/models/atlas_engine/address_validation/token/comparison.rb

@@ -0,0 +1,76 @@
+# typed: true
+# frozen_string_literal: true
+
+module AtlasEngine
+  module AddressValidation
+    class Token
+      class Comparison
+        extend T::Sig
+        include Comparable
+
+        QUALIFIERS = T.let(
+          [:equal, :prefix, :suffix, :comp].freeze,
+          T::Array[Symbol],
+        )
+
+        sig { returns(Token) }
+        attr_reader :left, :right
+
+        sig { returns(Symbol) }
+        attr_reader :qualifier
+
+        sig { returns(Integer) }
+        attr_reader :edit_distance
+
+        sig do
+          params(
+            left: Token,
+            right: Token,
+            qualifier: Symbol,
+            edit_distance: Integer,
+          ).void
+        end
+        def initialize(left:, right:, qualifier:, edit_distance:)
+          raise "Unknown qualifier" unless qualifier.in?(QUALIFIERS)
+
+          @left = left
+          @right = right
+          @qualifier = qualifier
+          @edit_distance = edit_distance
+        end
+
+        sig { params(other: Comparison).returns(Integer) }
+        def <=>(other)
+          if edit_distance == other.edit_distance
+            qualifier_rank <=> other.qualifier_rank
+          else
+            edit_distance <=> other.edit_distance
+          end
+        end
+
+        sig { returns(T::Boolean) }
+        def equal?
+          qualifier == :equal
+        end
+
+        sig { params(other: Comparison).returns(T::Boolean) }
+        def preceeds?(other)
+          left.preceeds?(other.left) && right.preceeds?(other.right)
+        end
+
+        sig { returns(String) }
+        def inspect
+          "<comp left:#{left.inspect} #{qualifier.to_s.upcase} right:#{right.inspect} edit:#{edit_distance}/>"
+        end
+
+        protected
+
+        sig { returns(Integer) }
+        def qualifier_rank
+          # Constructor verifies that qualifier is in the list
+          T.must(QUALIFIERS.find_index(qualifier))
+        end
+      end
+    end
+  end
+end

data/app/models/atlas_engine/address_validation/token/sequence/comparator.rb

@@ -0,0 +1,158 @@
+# typed: true
+# frozen_string_literal: true
+
+module AtlasEngine
+  module AddressValidation
+    class Token
+      class Sequence
+        class Comparator
+          extend T::Sig
+
+          sig { returns(Sequence) }
+          attr_reader :left, :right
+
+          attr_reader :comparison_cache
+
+          MAX_ALLOWED_EDIT_DISTANCE_PERCENT = 0.5
+
+          sig { params(left_sequence: Sequence, right_sequence: Sequence).void }
+          def initialize(left_sequence:, right_sequence:)
+            @left = left_sequence
+            @right = right_sequence
+            @comparison_cache = Hash.new do |h, (l_tok, r_tok)|
+              h[[l_tok, r_tok]] = AddressValidation::Token::Comparator.new(l_tok, r_tok).compare
+            end
+          end
+
+          sig { returns(Comparison) }
+          def compare
+            result = left.permutations.product(right.permutations).map do |left_permutation, right_permutation|
+              flattened_sequence_compare(left_permutation, right_permutation)
+            end
+
+            T.must(result.min)
+          end
+
+          private
+
+          sig do
+            params(
+              left_permutations: T::Array[Token],
+              right_permutations: T::Array[Token],
+            ).returns(T::Array[Token::Comparison])
+          end
+          def token_comparisons(left_permutations, right_permutations)
+            left_permutations.product(right_permutations).map do |l_tok, r_tok|
+              comparison_cache[[l_tok, r_tok]]
+            end
+          end
+
+          sig do
+            params(
+              token_comparisons: T::Array[Token::Comparison],
+            ).returns(T::Array[Token::Comparison])
+          end
+          def sort_token_comparisons(token_comparisons)
+            token_comparisons.sort do |a, b|
+              comp = a <=> b
+
+              if comp == 0
+                (a.left.position + a.right.position) <=> (b.left.position + b.right.position)
+              else
+                comp
+              end
+            end
+          end
+
+          sig do
+            params(
+              left_permutation: T::Array[Token],
+              right_permutation: T::Array[Token],
+            ).returns(Sequence::Comparison)
+          end
+          def flattened_sequence_compare(left_permutation, right_permutation)
+            token_comparisons = token_comparisons(left_permutation, right_permutation)
+            sorted_token_comparisons = sort_token_comparisons(token_comparisons)
+
+            filtered_token_comparisons = []
+
+            until sorted_token_comparisons.empty?
+              closest_match = sorted_token_comparisons.shift
+
+              if tokens_match_by_edit_distance?(comparison: T.must(closest_match))
+                filtered_token_comparisons << closest_match
+              end
+
+              sorted_token_comparisons.delete_if do |comparison|
+                same_token_or_position?(comparison.left, T.must(closest_match).left) ||
+                  same_token_or_position?(comparison.right, T.must(closest_match).right)
+              end
+            end
+
+            sorted_token_comparisons = filtered_token_comparisons.sort do |token1, token2|
+              token1.left.position <=> token2.left.position
+            end
+
+            Comparison.new(
+              unmatched_tokens: unmatched_tokens(left_permutation, right_permutation, sorted_token_comparisons),
+              token_comparisons: sorted_token_comparisons,
+              left_sequence: left,
+              right_sequence: right,
+            )
+          end
+
+          sig { params(comparison: AddressValidation::Token::Comparison).returns(T::Boolean) }
+          def tokens_match_by_edit_distance?(comparison:)
+            max_edit_distance = [comparison.left.value.length, comparison.right.value.length].max
+            edit_distance_percent = comparison.edit_distance.to_f / max_edit_distance
+
+            :prefix == comparison.qualifier || edit_distance_percent <= MAX_ALLOWED_EDIT_DISTANCE_PERCENT
+          end
+
+          sig do
+            params(
+              left_tokens: T::Array[Token],
+              right_tokens: T::Array[Token],
+              comparisons: T::Array[Token::Comparison],
+            ).returns(T::Array[Token])
+          end
+          def unmatched_tokens(left_tokens, right_tokens, comparisons)
+            remaining_left_tokens = left_tokens.reject do |token|
+              comparisons.any? do |comparison|
+                same_token_or_position?(comparison.left, token)
+              end
+            end
+
+            remaining_left_tokens = remove_synonyms_at_same_position(remaining_left_tokens)
+
+            remaining_right_tokens = right_tokens.reject do |token|
+              comparisons.any? do |comparison|
+                same_token_or_position?(comparison.right, token)
+              end
+            end
+
+            remaining_right_tokens = remove_synonyms_at_same_position(remaining_right_tokens)
+
+            remaining_left_tokens.concat(remaining_right_tokens)
+          end
+
+          sig { params(token: Token, other_token: Token).returns(T::Boolean) }
+          def same_token_or_position?(token, other_token)
+            return true if token == other_token
+
+            token.offset_range == other_token.offset_range && token.position == other_token.position
+          end
+
+          sig { params(tokens: T::Array[Token]).returns(T::Array[Token]) }
+          def remove_synonyms_at_same_position(tokens)
+            tokens.group_by(&:position)
+              .each do |_, tokens|
+                tokens.reject! { |token| token.type == "SYNONYM" } if tokens.size > 1
+              end
+              .values.flatten
+          end
+        end
+      end
+    end
+  end
+end

data/app/models/atlas_engine/address_validation/token/sequence/comparison.rb

@@ -0,0 +1,166 @@
+# typed: true
+# frozen_string_literal: true
+
+module AtlasEngine
+  module AddressValidation
+    class Token
+      class Sequence
+        class Comparison
+          extend T::Sig
+          include Comparable
+
+          DEFAULT_PARTIAL_MATCH_THRESHOLD_PERCENT = 0.5
+
+          attr_reader :unmatched_tokens, :left_sequence, :right_sequence, :token_comparisons
+
+          sig do
+            params(
+              unmatched_tokens: T::Array[Token],
+              token_comparisons: T::Array[Token::Comparison],
+              left_sequence: T.nilable(Sequence),
+              right_sequence: T.nilable(Sequence),
+            ).void
+          end
+          def initialize(unmatched_tokens:, token_comparisons:, left_sequence:, right_sequence:)
+            @unmatched_tokens = unmatched_tokens
+            @token_comparisons = token_comparisons
+            @left_sequence = left_sequence
+            @right_sequence = right_sequence
+          end
+
+          sig { params(other: Comparison).returns(Integer) }
+          def <=>(other)
+            # > num matches
+            # longest subsequence
+            # < num unmatched (kinda related to < aggregate edit distance)
+            # < aggregate edit distance
+            # > num prefixes
+            # > num suffixes
+            matches = count_by_qualifier(:equal) <=> other.count_by_qualifier(:equal)
+            return matches * -1 if matches.nonzero?
+
+            unmatched = unmatched_tokens.size <=> other.unmatched_tokens.size
+            return unmatched if unmatched.nonzero?
+
+            longest_subsequence = longest_subsequence_comparison <=> other.longest_subsequence_comparison
+            return -1 * longest_subsequence if longest_subsequence.nonzero?
+
+            edit_distance = aggregate_edit_distance <=> other.aggregate_edit_distance
+            return edit_distance if edit_distance.nonzero?
+
+            prefixes = count_by_qualifier(:prefix) <=> other.count_by_qualifier(:prefix)
+            return prefixes * -1 if prefixes.nonzero?
+
+            (count_by_qualifier(:suffix) <=> other.count_by_qualifier(:suffix)) * -1
+          end
+
+          sig { returns(String) }
+          def inspect
+            parts = ["["]
+            token_comparisons.each do |comparison|
+              parts << "\n#{comparison.inspect}"
+            end
+            parts << "\n" unless token_comparisons.empty?
+            parts << "]"
+            "<seqcomp unmatched:#{unmatched_tokens.inspect} comp:#{parts.join}/>"
+          end
+
+          sig { params(other_comparison: Comparison).returns(T::Boolean) }
+          def better_than?(other_comparison)
+            self < other_comparison
+          end
+
+          sig { params(other_comparison: Comparison).returns(T::Boolean) }
+          def worse_than?(other_comparison)
+            self > other_comparison
+          end
+
+          sig { params(other_comparison: Comparison).returns(T::Boolean) }
+          def equivalent_to?(other_comparison)
+            self == other_comparison
+          end
+
+          sig { params(other_comparison: Comparison).returns(Comparison) }
+          def merge(other_comparison)
+            AddressValidation::Token::Sequence::Comparison.new(
+              unmatched_tokens: unmatched_tokens + other_comparison.unmatched_tokens,
+              token_comparisons: (token_comparisons + other_comparison.token_comparisons).uniq,
+              left_sequence: left_sequence.equal?(other_comparison.left_sequence) ? left_sequence : nil,
+              right_sequence: right_sequence.equal?(other_comparison.right_sequence) ? right_sequence : nil,
+            )
+          end
+
+          sig { returns(T::Boolean) }
+          def match?
+            aggregate_edit_distance == 0 && unmatched_tokens.empty?
+          end
+
+          sig { params(threshold_percent: Float).returns(T::Boolean) }
+          def potential_match?(threshold_percent: DEFAULT_PARTIAL_MATCH_THRESHOLD_PERCENT)
+            matched_tokens_percent >= threshold_percent && matched_length_percent >= threshold_percent
+          end
+
+          sig { returns(Integer) }
+          def aggregate_edit_distance
+            token_comparisons.sum(&:edit_distance)
+          end
+
+          sig { returns(Integer) }
+          def token_match_count
+            token_comparisons.size
+          end
+
+          protected
+
+          sig { params(qualifier: Symbol).returns(Integer) }
+          def count_by_qualifier(qualifier)
+            token_comparisons.count { |comparison| comparison.qualifier == qualifier }
+          end
+
+          sig { returns([Integer, Integer]) }
+          def longest_subsequence_comparison
+            max_subsequence_length = subsequence_lengths.max || 0
+            # max length, number of times we saw a subsequence of max length (acts as a tiebreaker)
+            [max_subsequence_length, subsequence_lengths.count(max_subsequence_length)]
+          end
+
+          private
+
+          sig { returns(T::Array[Integer]) }
+          def subsequence_lengths
+            # measure length of consecutive pairs of equal tokens. The position of both compared tokens
+            # must increase by 1 relative to the preceeding AddressValidation::Token::Comparison's pair.
+            @subsequence_lengths = equal_token_comparisons
+              .chunk_while { |token_comp, next_token_comp| token_comp.preceeds?(next_token_comp) }
+              .map(&:length)
+              .select { |length| length > 1 } # trivial sequences of length 1 are ignored
+          end
+
+          sig { returns(T::Array[Token::Comparison]) }
+          def equal_token_comparisons
+            token_comparisons.select(&:equal?)
+          end
+
+          sig { returns(Float) }
+          def matched_tokens_percent
+            matched_tokens_count = token_comparisons.size * 2
+            unmatched_tokens_count = unmatched_tokens.size
+            (matched_tokens_count.to_f / (matched_tokens_count + unmatched_tokens_count)).round(2)
+          end
+
+          sig { returns(Float) }
+          def matched_length_percent
+            matched_length = token_comparisons.sum do |token_pair|
+              token_pair.left.value.length + token_pair.right.value.length - token_pair.edit_distance
+            end
+            total_edit_distance = token_comparisons.sum(&:edit_distance)
+            unmatched_length = unmatched_tokens.sum do |token|
+              token.value.length
+            end
+            (matched_length.to_f / (matched_length + unmatched_length + total_edit_distance)).round(2)
+          end
+        end
+      end
+    end
+  end
+end

data/app/models/atlas_engine/address_validation/token/sequence.rb

@@ -0,0 +1,147 @@
+# typed: true
+# frozen_string_literal: true
+
+module AtlasEngine
+  module AddressValidation
+    class Token
+      class Sequence
+        extend T::Sig
+
+        class << self
+          extend T::Sig
+          include Normalizer
+          ACCEPTABLE_CHARACTERS = /\p{Alnum}/
+
+          sig { params(string: T.nilable(String)).returns(Sequence) }
+          def from_string(string)
+            start_offset = 0
+            end_offset = 0
+            position = 0
+
+            tokens = Annex29.segment_words(string).filter_map do |substring|
+              start_offset = end_offset
+              end_offset = start_offset + substring.length
+
+              normalized_substring = normalize(substring)
+              # annex 29 returns whitespace and punctuation as separate substrings
+              next unless normalized_substring.match?(ACCEPTABLE_CHARACTERS)
+
+              token = Token.new(
+                value: normalized_substring,
+                start_offset: start_offset,
+                end_offset: end_offset,
+                position: position,
+                type: number?(substring) ? "<NUM>" : "<ALPHANUM>",
+              )
+
+              position += 1
+
+              token
+            end
+
+            new(tokens: tokens, raw_value: string)
+          end
+
+          def number?(string)
+            !Float(string).nil?
+          rescue
+            false
+          end
+        end
+
+        TokenOrSynonyms = T.type_alias { T.any(Token, Synonyms) }
+
+        sig { returns(T::Array[TokenOrSynonyms]) }
+        attr_reader :tokens
+
+        sig { returns(T.nilable(String)) }
+        attr_reader :raw_value
+
+        # Sorbet can't handle delegates https://github.com/sorbet/sorbet/issues/4794
+        # rubocop:disable Rails/Delegate
+        sig { returns(T::Boolean) }
+        def empty? = tokens.empty?
+
+        sig { returns(Integer) }
+        def size = tokens.size
+
+        sig { returns(Integer) }
+        def length = tokens.length
+        # rubocop:enable Rails/Delegate
+
+        sig { params(tokens: T::Array[Token], raw_value: T.nilable(String)).void }
+        def initialize(tokens: [], raw_value: nil)
+          @raw_value = raw_value
+          @tokens = group_by_overlapping_offsets(tokens)
+            .map { |tkns| tkns.one? ? T.must(tkns.first) : Synonyms.new(tokens: tkns) }
+        end
+
+        sig { returns(String) }
+        def inspect
+          "<seq #{tokens.inspect}/>"
+        end
+
+        sig { returns(T::Array[T::Array[Token]]) }
+        def permutations = recursive_permutations(tokens)
+
+        def ==(other)
+          return false unless other.is_a?(Sequence)
+
+          tokens == other.tokens
+        end
+
+        private
+
+        sig { params(token_array: T::Array[TokenOrSynonyms]).returns(T::Array[T::Array[Token]]) }
+        def recursive_permutations(token_array)
+          # we bottom out when token_array contains only simple tokens
+          next_synonyms_index = token_array.find_index { |entry| entry.is_a?(Synonyms) }
+          # There are no synonyms in that array, cast is safe
+          return [T.cast(token_array, T::Array[Token])] unless next_synonyms_index
+
+          new_tokens = token_array.dup
+          synonyms = T.cast(new_tokens[next_synonyms_index], Synonyms)
+          new_tokens.delete_at(next_synonyms_index)
+
+          if synonyms.multi_token?
+            # token_array (before synonyms object was deleted):  [a, b, <syn [afb, [air, force, base]]/>, ...rest]
+            # output: [[a, b, afb, ...rest], [a, b, air, force, base, ...rest]]
+            synonyms.tokens.flat_map do |multi_token_entry|
+              current_permutation = T.unsafe(new_tokens).dup.insert(next_synonyms_index, *Array(multi_token_entry))
+              # ...rest will be handled recursively
+              recursive_permutations(current_permutation)
+            end
+          else
+            # token_array (before synonyms object was deleted):  [a, b, <syn [st, street, saint/>, ...rest]
+            # output: [[a, b, st, street, saint, ...rest]]
+            T.unsafe(new_tokens).insert(next_synonyms_index, *synonyms.tokens)
+            # ...rest will be handled recursively
+            recursive_permutations(new_tokens)
+          end
+        end
+
+        sig { params(tokens: T::Array[Token]).returns(T::Array[T::Array[Token]]) }
+        def group_by_overlapping_offsets(tokens)
+          return [] if tokens.empty?
+
+          sorted_tokens = tokens.stable_sort_by(&:position)
+          current_range = sorted_tokens.first&.offset_range
+
+          groups = []
+          current_group = []
+          sorted_tokens.each do |token|
+            if current_range.cover?(token.offset_range)
+              current_group << token
+            else
+              groups << current_group
+              current_group = [token]
+              current_range = token.offset_range
+            end
+          end
+
+          groups << current_group
+        end
+      end
+    end
+  end
+end

data/app/models/atlas_engine/address_validation/token/synonyms.rb

@@ -0,0 +1,77 @@
+# typed: true
+# frozen_string_literal: true
+
+module AtlasEngine
+  module AddressValidation
+    class Token
+      class Synonyms
+        extend T::Sig
+        # Similar to a Token, and has some of the same methods like position, value, type.
+
+        TokenList = T.type_alias { T::Array[Token] }
+
+        sig { returns(T::Array[T.any(Token, TokenList)]) }
+        attr_reader :tokens
+
+        # Sorbet can't handle delegates https://github.com/sorbet/sorbet/issues/4794
+        # rubocop:disable Rails/Delegate
+        sig { returns(Integer) }
+        def position = first_token.position
+
+        sig { returns(T::Range[Integer]) }
+        def offset_range = first_token.offset_range
+        # rubocop:enable Rails/Delegate
+
+        sig { params(tokens: T::Array[Token]).void }
+        def initialize(tokens: [])
+          raise ArgumentError, "Synonyms cannot be empty" if tokens.empty?
+
+          @tokens = []
+          tokens_by_position = tokens.stable_sort_by(&:position).group_by(&:position)
+
+          while tokens_by_position.values.any?(&:present?)
+            current_group = []
+            starting_position = tokens_by_position.keys.first
+
+            while tokens_by_position.key?(starting_position)
+              token = T.must(tokens_by_position[starting_position]).shift
+              current_group << token
+              starting_position += T.must(token).position_length
+            end
+
+            @tokens << (current_group.one? ? current_group.first : current_group)
+            tokens_by_position.compact_blank! # remove positions having no tokens
+          end
+        end
+
+        sig { returns(String) }
+        def inspect
+          "<syn #{tokens.inspect}/>"
+        end
+
+        sig { returns(NilClass) }
+        def value
+          nil
+        end
+
+        sig { returns(String) }
+        def type
+          "<SYNONYMS>"
+        end
+
+        sig { returns(T::Boolean) }
+        def multi_token?
+          tokens.any?(Array)
+        end
+
+        private
+
+        sig { returns(Token) }
+        def first_token
+          head = T.must(tokens.first)
+          head.is_a?(Array) ? T.must(head.first) : head
+        end
+      end
+    end
+  end
+end