eco-helpers 2.0.16 → 2.0.17

data/lib/eco/data/fuzzy_match/ngrams_score.rb

@@ -0,0 +1,73 @@
+module Eco
+  module Data
+    module FuzzyMatch
+      module NGramsScore
+        # It does the following:
+        #   1. It splits both strings into words
+        #   2. Pairs all words by best `ngrams_score` match
+        #   3. Gives `0` score to those words of `str2` that lost their pair (a word of `str1` cannot be paired twice)
+        #   4. Merges the `ngrams_score` of all the paired words of `str2` against their `str1` word pair
+        # @param range [Integer, Range] determine the lenght of the generated values for each `word`.
+        # @normalized [Boolean] to avoid double ups in normalizing.
+        # @return [Score] the score object with the result.
+        def words_ngrams_score(str1, str2, range: 3..5, normalized: false)
+          str1, str2 = normalize_string([str1, str2]) unless normalized
+          len1 = str1 && str1.length; len2 = str2 && str2.length
+
+          Score.new(0, 0).tap do |score|
+            next if !str2 || !str1
+            next score.increase(score.total) if str1 == str2
+            next if str1.length < 2 || str1.length < 2
+
+            paired_words(str1, str2, normalized: true) do |needle, item|
+              ngrams_score(needle, item, range: range, normalized: true)
+            end.each do |sub_str1, (item, iscore)|
+              #puts "pairs '#{sub_str1}' --> '#{item}' (score: #{iscore.ratio})"
+              score.merge!(iscore)
+            end
+          end
+        end
+
+        # A score is kept of matching ngram combinations of `str2`.
+        # @note This algorithm is best suited for matching sentences, or 'firstname lastname' compared with 'lastname firstname' combinations.
+        # @param range [Integer, Range] determine the lenght of the generated values.
+        # @normalized [Boolean] to avoid double ups in normalizing.
+        # @return [Score] the score object with the result.
+        def ngrams_score(str1, str2, range: 3..5, normalized: false)
+          str1, str2 = normalize_string([str1, str2]) unless normalized
+          len1 = str1 && str1.length; len2 = str2 && str2.length
+
+          Score.new(0, len1 || 0).tap do |score|
+            next if !str2 || !str1
+            next score.increase(score.total) if str1 == str2
+            next if str1.length < 2 || str2.length < 2
+
+            grams = word_ngrams(str2, range, normalized: true)
+            next unless grams.length > 0
+
+            if range.is_a?(Integer)
+              item_weight = score.total.to_f / grams.length
+              matches     = grams.select {|res| str1.include?(gram)}.length
+              score.increase(matches * item_weight)
+            else
+              groups       = grams.group_by {|gram| gram.length}
+              sorted_lens  = groups.keys.sort.reverse
+              lens         = sorted_lens.length
+              group_weight = (1.0 / lens).round(3)
+
+              groups.each do |len, grams|
+                len_max_score  = score.total * group_weight
+                item_weight    = len_max_score / grams.length
+                matches        = grams.select {|gram| str1.include?(gram)}.length
+                #pp "#{len} match: #{matches} (over #{grams.length}) || max_score: #{len_max_score} (over #{score.total})"
+                score.increase(matches * item_weight)
+              end
+            end
+
+          end
+        end
+
+      end
+    end
+  end
+end

data/lib/eco/data/fuzzy_match/pairing.rb

@@ -0,0 +1,102 @@
+module Eco
+  module Data
+    module FuzzyMatch
+      module Pairing
+
+        # Pair words using some algorithm.
+        # It does the following:
+        #   1. It splits both strings into words.
+        #   2. Pairs all words by using `block` to score the best match.
+        #   3. Gives `0` score to those words of `str2` that lost their pair (a word of `str1` cannot be paired twice).
+        #   4. Merges the `Score` of all the paired words of `str2` against their `str1` word pair.
+        # @yield [needle, item] offers a comparison algorithm between two strings.
+        # @yieldparam needle [String] the string of reference.
+        # @yieldparam item [String] one of the haystack items.
+        # @yieldreturn [Eco::Data::FuzzyMatch::Score] the `Score` object with the results of comparing `str1` and `str2`
+        # @param str1 [String] the string of reference.
+        # @param str2 [String] one of the haystack items.
+        # @param format [Symbol] determines the `values` of the returned `Hash`::
+        #   1. `:pair` for just pair
+        #   2. `:score` for just score
+        #   2. `[:pair, :score]` for `Array`
+        # @normalized [Boolean] to avoid double ups in normalizing.
+        # @return [Hash] where `keys` are the **words** of `str1` and their `values`:
+        #   1. if `format` is `:pair` => the `str2` words with highest match.
+        #   2. if `format` is `:score` => the `Score` words with highest match.
+        #   3. if `format` is `[:pair, :score]` => both in an `Array`.
+        def paired_words(str1, str2, format: [:pair, :score], normalized: false)
+          str1, str2 = normalize_string([str1, str2]) unless normalized
+          return {} if !str2 || !str1
+          return score.increase(score.total) if str1 == str2
+          return {str1 => nil} if str1.length < 2 || str1.length < 2
+
+          needles    = get_words(str1, normalized: true)
+          haystack   = get_words(str2, normalized: true)
+
+          ranking    = {}
+          faceted    = needles.each_with_object({}) do |needle, faceted|
+            faceted[needle] = haystack.map do |item|
+              {
+                pair:  item,
+                score: yield(needle, item)
+              }.tap do |result|
+                ranking[item] ||= []
+                if result[:score].ratio > 0.05
+                  ranking[item] << ({needle: needle, score: result[:score]})
+                end
+              end
+            end.sort_by do |result|
+              result[:score].ratio
+            end.reverse
+          end
+
+          paired = {}
+          #scores = {}
+          ranking.each do |item, results|
+            sorted = results.reject do |result|
+              paired.key?(result[:needle])
+            end.sort_by do |result|
+              result[:score].ratio
+            end.reverse
+            if result = sorted.shift
+              paired[result[:needle]] = {
+                pair:  item,
+                score: result[:score]
+              }
+            end
+          end
+
+          pending_items   = haystack - paired.values
+          faceted.reject do |needle, results|
+            paired.key?(needle)
+          end.each do |needle, results|
+            results.select! do |result|
+              pending_items.include?(result[:pair]) && result[:score].ratio > 0.05
+            end
+            if result = results.shift
+              paired[needle] = result
+              pending_items.delete(result[:pair])
+            end
+          end
+
+          pending_needles = needles - paired.keys
+          pending_needles.each do |needle|
+            paired[needle] = {
+              pair:  nil,
+              score: Score.new(0, needle.length)
+            }
+          end
+          paired.transform_values do |result|
+            case format
+            when Array
+              result.values_at(*format)
+            else
+              restult[format]
+            end
+          end
+        end
+
+      end
+    end
+  end
+end

data/lib/eco/data/fuzzy_match/result.rb

@@ -0,0 +1,67 @@
+module Eco
+  module Data
+    module FuzzyMatch
+      class Result < Struct.new(:match, :value, :dice, :levenshtein, :jaro_winkler, :ngrams, :words_ngrams, :chars_position)
+        ALL_METHODS = [:dice, :levenshtein, :jaro_winkler, :ngrams, :words_ngrams, :chars_position]
+
+        def dice; super&.round(3); end
+        def levenshtein; super&.round(3); end
+        def jaro_winkler; super&.round(3); end
+        def ngrams; super&.round(3); end
+        def words_ngrams; super&.round(3); end
+        def chars_position; super&.round(3); end
+
+        # TODO: print in the order of `order`
+        def print
+          msg  = "(Dice: #{dice}) (Lev Dst: #{levenshtein}) "
+          msg << "(Jaro: #{jaro_winkler}) "
+          msg << "(Ngram: #{ngrams}) (WNgrams: #{words_ngrams}) "
+          msg << "(C Pos: #{chars_position}) "
+          msg << "'#{value}'"
+        end
+
+        def all_threshold?(methods = order, threshold = 0.15)
+          return true unless threshold
+          [methods].flatten.compact.all? {|method| threshold?(method, threshold)}
+        end
+
+        def any_threshold?(methods = order, threshold = 0.15)
+          return true unless threshold
+          [methods].flatten.compact.any? {|method| threshold?(method, threshold)}
+        end
+
+        def threshold?(method = :dice, threshold = 0.15)
+          raise "Uknown method '#{method}'" unless self.respond_to?(method)
+          self.send(method) >= threshold
+        end
+
+        def order=(values)
+          @order = [values].flatten.compact.tap do |o|
+            o = [:words_ngrams, :dice] if o.empty?
+          end
+        end
+
+        def order
+          @order ||= [:words_ngrams, :dice]
+        end
+
+        def <=>(result)
+          compare(result)
+        end
+
+        private
+
+        def compare(other, order: self.order)
+          return 0  unless method = order.first
+          raise "Uknown method '#{method}'"  unless self.respond_to?(method) && other.respond_to?(method)
+          return -1 if self.send(method) > other.send(method)
+          return  1 if self.send(method) < other.send(method)
+          compare(other, order: order[1..-1])
+        end
+
+
+      end
+
+    end
+  end
+end

data/lib/eco/data/fuzzy_match/results.rb

@@ -0,0 +1,53 @@
+module Eco
+  module Data
+    module FuzzyMatch
+      class Results < Struct.new(:needle, :value, :raw_results)
+
+        def results_with_false_positives
+          relevant_results(methods: :jaro_winkler, threshold: 0.5)
+        end
+
+        def relevant_results(methods: order, threshold: 0.5)
+          raw_results.select do |result|
+            result.all_threshold?(methods, threshold)
+          end.yield_self do |filtered|
+            self.class.new(needle, value, filtered).tap do |results|
+              results.order = methods
+            end
+          end
+        end
+
+        def order=(values)
+          @order = [values].flatten.compact
+          raw_results.each {|r| r.order = @order}
+        end
+
+        def order
+          @order ||= [:words_ngrams, :dice]
+        end
+
+        def results
+          raw_results.sort
+        end
+
+        def print
+          msg = results.map do |result|
+            result.print
+          end.join("\n  ")
+
+          puts "'#{value}':\n  " + msg
+        end
+
+        private
+
+        def item_string(item, attr = self.method)
+          return item if !item || item.is_a?(String) || !attr
+          attr = attr.to_sym
+          return item.send(attr) if item.respond_to?(attr)
+        end
+
+      end
+
+    end
+  end
+end

data/lib/eco/data/fuzzy_match/score.rb

@@ -0,0 +1,44 @@
+module Eco
+  module Data
+    module FuzzyMatch
+      class Score < Struct.new(:score, :total)
+
+        def ratio(decimals = 6)
+          ((score || 0).to_f / (total || 1)).round(decimals)
+        end
+
+        def percent(decimals = 3)
+          (100 * ratio).round(decimals)
+        end
+
+        def increase(value = 1)
+          self.score += value
+        end
+
+        def increase_total(value)
+          self.total += value
+        end
+
+        def values_at(*keys)
+          keys.map do |key|
+            self.send(key) if self.respond_to?(key)
+          end
+        end
+
+        # Merges 2 Score instance objects
+        def merge(value)
+          Score.new(*values_at(:score, :total)).merge!(value)
+        end
+
+        def merge!(value)
+          raise "Expecting Score object. Given: #{value.class}" unless value.is_a?(Score)
+          increase(value.score)
+          increase_total(value.total)
+          self
+        end
+
+      end
+
+    end
+  end
+end

data/lib/eco/data/fuzzy_match/stop_words.rb

@@ -0,0 +1,35 @@
+module Eco
+  module Data
+    module FuzzyMatch
+      module StopWords
+        PREPOSITIONS = [
+          "aboard", "about", "above", "across", "after", "against", "along", "amid", "among", "around", "as", "at",
+          "before", "behind", "below", "beneath", "beside", "between", "beyond", "but", "by",
+          "concerning", "considering", "despite", "down", "during", "except", "following", "for", "from",
+          "in", "inside", "into", "like", "minus", "near", "next",
+          "of", "off", "on", "onto", "opposite", "out", "outside", "over", "past", "per", "plus",
+          "regarding", "round", "save", "since", "than", "through", "till", "to", "toward",
+          "under", "underneath", "unlike", "until", "up", "upon", "versus", "via",
+          "with", "within", "without"
+        ]
+        PRONOUNS = [
+          "all", "another", "any", "anybody", "anyone", "anything", "as", "aught",
+          "both", "each", "each other", "either", "enough", "everybody", "everyone", "everything",
+          "few", "he", "her", "hers", "herself", "him", "himself", "his", "I", "idem", "it", "its", "itself",
+          "many", "me", "mine", "most", "my", "myself", "naught", "neither", "no one", "nobody", "none", "nothing", "nought",
+          "one", "one another", "other", "others", "ought", "our", "ours", "ourself", "ourselves",
+          "several", "she", "some", "somebody", "someone", "something", "somewhat", "such", "suchlike",
+          "that", "thee", "their", "theirs", "theirself", "theirselves", "them", "themself", "themselves", "there",
+          "these", "they", "thine", "this", "those", "thou", "thy", "thyself", "us",
+          "we", "what", "whatever", "whatnot", "whatsoever", "whence", "where", "whereby", "wherefrom",
+          "wherein", "whereinto", "whereof", "whereon", "wherever", "wheresoever", "whereto", "whereunto",
+          "wherewith", "wherewithal", "whether", "which", "whichever", "whichsoever", "who", "whoever", "whom",
+          "whomever", "whomso", "whomsoever", "whose", "whosever", "whosesoever", "whoso", "whosoever",
+          "ye", "yon", "yonder", "you", "your", "yours", "yourself", "yourselves"
+        ]
+        ARTICLES = ["a", "an", "the"]
+
+      end
+    end
+  end
+end

data/lib/eco/data/fuzzy_match/string_helpers.rb

@@ -0,0 +1,69 @@
+module Eco
+  module Data
+    module FuzzyMatch
+      module StringHelpers
+        # Downcases and trims
+        def normalize_string(value)
+          case value
+          when Array
+            value.map {|val| normalize_string(val)}
+          when Symbol
+            normalize_string(value.to_sym)
+          when String
+            value.downcase.strip
+          end
+        end
+
+        def get_words(str, normalized: false)
+          return [] unless str
+          str = normalize_string(str) unless normalized
+          str.scan(/[a-zA-Z'-]+/)
+        end
+
+        # Keeps the start order of the `words` and consecutive `words` together/consecutive.
+        # @param str [String] the input string with the words.
+        # @param range [Integer, Range] determine the lenght of the generated values.
+        # @return [Array<String>] combinations of `range` length of `words`.
+        def string_ngrams(str, range=2..3, normalized: false)
+          ngrams(get_words(str, normalized: normalized), range)
+        end
+
+        # Keeps the start order of the `words` of the input `Array` `words`.
+        # It does **not** keep consecutive `words` together (it can jump/skip items).
+        # @param str [String] the input string with the words.
+        # @param range [Integer, Range] determine the lenght of the generated values.
+        # @return [Array<String>] combinations of `range` length of `words`
+        def string_combinations(str, range=2..3, normalized: false)
+          combinations(get_words(str, normalized: normalized), range)
+          .map {|comb| comb.join(' ')}
+        end
+
+        # It includes `combinations` that break the initial order of the `Array`.
+        # It does **not** keep consecutive `words` together (it can jump/skip items).
+        # @param str [String] the input string with the words.
+        # @param range [Integer, Range] determine the lenght of the generated values.
+        # @return [Array<String>] permutations of `range` length of `words`
+        def string_permutations(str, range=2..3, normalized: false)
+          permutations(get_words(str, normalized: normalized), range)
+          .map {|comb| comb.join(' ')}
+        end
+
+        # Keeps the start order of the `charts` and consecutive `charts` together/consecutive.
+        # @param str [String] the input `word` string.
+        # @param range [Integer, Range] determine the lenght of the generated values.
+        # @return [Array<String>] combinations of `range` length of `words`.
+        def word_ngrams(str, range=2..3, normalized: false)
+          str = normalize_string(str) unless normalized
+          ngrams(str.to_s.chars, range)
+          .map {|comb| no_blanks(comb)}
+        end
+
+        def no_blanks(str)
+          return nil unless str && str.is_a?(String)
+          str.tr(' ', '')
+        end
+
+      end
+    end
+  end
+end