eco-helpers 2.0.14 → 2.0.19

data/lib/eco/data/fuzzy_match/array_helpers.rb

@@ -0,0 +1,75 @@
+module Eco
+  module Data
+    module FuzzyMatch
+      module ArrayHelpers
+        # Keeps the start order of the `values` and consecutive `values` together/consecutive.
+        # @param values [Array] the input array with the values.
+        # @param range [Integer, Range] determine the lenght of the generated values.
+        # @return [Array<Array<Value>>] combinations of `range` length of `values`.
+        def ngrams(values, range=2..3)
+          [].tap do |out|
+            if range.is_a?(Integer)
+              n           = range
+              values_count = values.length
+              values.each_with_index do |word, i|
+                min = i
+                max = i + (n - 1)
+                break if values_count <= max
+                out << values[min..max].join(' ')
+              end
+              out.uniq!
+            else
+              range.each {|n| out.concat(ngrams(values, n))}
+              out.uniq!
+            end
+          end
+        end
+
+        # Keeps the start order of the `values` of the input `Array` `values`.
+        # It does **not** keep consecutive `values` together (it can jump/skip items).
+        # @param values [Array] the input array with the values.
+        # @param range [Integer, Range] determine the lenght of the generated values.
+        # @return [Array<Array<Value>>] combinations of `range` length of `values`
+        def combinations(values, range=2..3)
+          if range.is_a?(Integer)
+            values.combination(range).to_a
+          else
+            range.flat_map {|size| values.combination(size).to_a}
+          end
+        end
+
+        # It includes `combinations` that break the initial order of the `Array`.
+        # It does **not** keep consecutive `values` together (it can jump/skip items).
+        # @param values [Array] the input array with the values.
+        # @param range [Integer, Range] determine the lenght of the generated values.
+        # @return [Array<Array<Value>>] permutations of `range` length of `values`
+        def permutations(values, range=2..3)
+          combinations(values, range).tap do |out|
+            range = range.is_a?(Integer)? (range..range) : range
+            out.dup.select do |item|
+              range.include?(item.length)
+            end.each do |comb|
+              comb.permutation.to_a.tap do |perms|
+                perms.each {|perm| out << perm}
+              end
+            end
+            out.uniq!
+          end
+        end
+
+        # Helper to praper facet structure
+        # @param values1 [Array] the input array with the values to have their facet against.
+        # @param values2 [Array] the input array with the values to facet against.
+        # @return [Hash] where `keys` are `values1` and `value` of each `key` all `values2`
+        def facet(values1, values2)
+          {}.tap do |out|
+            next unless values1.is_a?(Enumerable)
+            values1 = values1.is_a?(Hash) ? values1.values : values1.to_a
+            values1.each {|val| out[val] = values2.dup}
+          end
+        end
+
+      end
+    end
+  end
+end

data/lib/eco/data/fuzzy_match/chars_position_score.rb

@@ -0,0 +1,37 @@
+module Eco
+  module Data
+    module FuzzyMatch
+      module CharsPositionScore
+        # For each character in `str1`, a search is performed on `str2`.
+        # The search is deemed successful if a character is found in `str2` within `max_distance` characters of the current position.
+        # A score is kept of matching characters.
+        # @note This algorithm is best suited for matching mis-spellings.
+        # @max_distance [Integer] maximum char position distance to score.
+        # @normalized [Boolean] to avoid double ups in normalizing.
+        # @return [Score] the score object with the result.
+        def chars_position_score(str1, str2, max_distance: 3, normalized: false)
+          str1, str2 = normalize_string([str1, str2]) unless normalized
+          len1 = str1 && str1.length; len2 = str2 && str2.length
+          Score.new(0, len1 || 0).tap do |score|
+            next if !str1 || !str2
+            next score.increase(score.total) if str1 == str2
+            next if len1 < 2
+            pos = 0
+            len1.times do |i|
+              start = pos + 1
+              found = false
+              if pos = str2.index(str1[i])
+                if pos < (start + max_distance)
+                  found = true
+                  score.increase
+                end
+              end
+              pos = start unless found
+            end
+          end
+        end
+
+      end
+    end
+  end
+end

data/lib/eco/data/fuzzy_match/ngrams_score.rb

@@ -0,0 +1,78 @@
+module Eco
+  module Data
+    module FuzzyMatch
+      module NGramsScore
+        # It does the following:
+        #   1. It splits both strings into words
+        #   2. Pairs all words by best `ngrams_score` match
+        #   3. Gives `0` score to those words of `str2` that lost their pair (a word of `str1` cannot be paired twice)
+        #   4. Merges the `ngrams_score` of all the paired words of `str2` against their `str1` word pair
+        # @param range [Integer, Range] determine the lenght of the generated values for each `word`.
+        # @normalized [Boolean] to avoid double ups in normalizing.
+        # @return [Score] the score object with the result.
+        def words_ngrams_score(str1, str2, range: 3..5, normalized: false)
+          str1, str2 = normalize_string([str1, str2]) unless normalized
+          len1 = str1 && str1.length; len2 = str2 && str2.length
+
+          Score.new(0, 0).tap do |score|
+            next if !str2 || !str1
+            if str1 == str2
+              score.increase_total(len1)
+              score.increase(score.total)
+            end
+            if str1.length < 2 || str1.length < 2
+              score.increase_total(len1)
+            end
+
+            paired_words(str1, str2, normalized: true) do |needle, item|
+              ngrams_score(needle, item, range: range, normalized: true)
+            end.each do |sub_str1, (item, iscore)|
+              #puts "pairs '#{sub_str1}' --> '#{item}' (score: #{iscore.ratio})"
+              score.merge!(iscore)
+            end
+          end
+        end
+
+        # A score is kept of matching ngram combinations of `str2`.
+        # @note This algorithm is best suited for matching sentences, or 'firstname lastname' compared with 'lastname firstname' combinations.
+        # @param range [Integer, Range] determine the lenght of the generated values.
+        # @normalized [Boolean] to avoid double ups in normalizing.
+        # @return [Score] the score object with the result.
+        def ngrams_score(str1, str2, range: 3..5, normalized: false)
+          str1, str2 = normalize_string([str1, str2]) unless normalized
+          len1 = str1 && str1.length; len2 = str2 && str2.length
+
+          Score.new(0, len1 || 0).tap do |score|
+            next if !str2 || !str1
+            next score.increase(score.total) if str1 == str2
+            next if str1.length < 2 || str2.length < 2
+
+            grams = word_ngrams(str2, range, normalized: true)
+            next unless grams.length > 0
+
+            if range.is_a?(Integer)
+              item_weight = score.total.to_f / grams.length
+              matches     = grams.select {|res| str1.include?(gram)}.length
+              score.increase(matches * item_weight)
+            else
+              groups       = grams.group_by {|gram| gram.length}
+              sorted_lens  = groups.keys.sort.reverse
+              lens         = sorted_lens.length
+              group_weight = (1.0 / lens).round(3)
+
+              groups.each do |len, grams|
+                len_max_score  = score.total * group_weight
+                item_weight    = len_max_score / grams.length
+                matches        = grams.select {|gram| str1.include?(gram)}.length
+                #pp "#{len} match: #{matches} (over #{grams.length}) || max_score: #{len_max_score} (over #{score.total})"
+                score.increase(matches * item_weight)
+              end
+            end
+
+          end
+        end
+
+      end
+    end
+  end
+end

data/lib/eco/data/fuzzy_match/pairing.rb

@@ -0,0 +1,101 @@
+module Eco
+  module Data
+    module FuzzyMatch
+      module Pairing
+
+        # Pair words using some algorithm.
+        # It does the following:
+        #   1. It splits both strings into words.
+        #   2. Pairs all words by using `block` to score the best match.
+        #   3. Gives `0` score to those words of `str2` that lost their pair (a word of `str1` cannot be paired twice).
+        #   4. Merges the `Score` of all the paired words of `str2` against their `str1` word pair.
+        # @yield [needle, item] offers a comparison algorithm between two strings.
+        # @yieldparam needle [String] the string of reference.
+        # @yieldparam item [String] one of the haystack items.
+        # @yieldreturn [Eco::Data::FuzzyMatch::Score] the `Score` object with the results of comparing `str1` and `str2`
+        # @param str1 [String] the string of reference.
+        # @param str2 [String] one of the haystack items.
+        # @param format [Symbol] determines the `values` of the returned `Hash`::
+        #   1. `:pair` for just pair
+        #   2. `:score` for just score
+        #   2. `[:pair, :score]` for `Array`
+        # @normalized [Boolean] to avoid double ups in normalizing.
+        # @return [Hash] where `keys` are the **words** of `str1` and their `values`:
+        #   1. if `format` is `:pair` => the `str2` words with highest match.
+        #   2. if `format` is `:score` => the `Score` words with highest match.
+        #   3. if `format` is `[:pair, :score]` => both in an `Array`.
+        def paired_words(str1, str2, format: [:pair, :score], normalized: false)
+          str1, str2 = normalize_string([str1, str2]) unless normalized
+          return {} if !str2 || !str1
+          return {str1 => nil} if str1.length < 2 || str1.length < 2
+
+          needles    = get_words(str1, normalized: true)
+          haystack   = get_words(str2, normalized: true)
+
+          ranking    = {}
+          faceted    = needles.each_with_object({}) do |needle, faceted|
+            faceted[needle] = haystack.map do |item|
+              {
+                pair:  item,
+                score: yield(needle, item)
+              }.tap do |result|
+                ranking[item] ||= []
+                if result[:score].ratio > 0.05
+                  ranking[item] << ({needle: needle, score: result[:score]})
+                end
+              end
+            end.sort_by do |result|
+              result[:score].ratio
+            end.reverse
+          end
+
+          paired = {}
+          #scores = {}
+          ranking.each do |item, results|
+            sorted = results.reject do |result|
+              paired.key?(result[:needle])
+            end.sort_by do |result|
+              result[:score].ratio
+            end.reverse
+            if result = sorted.shift
+              paired[result[:needle]] = {
+                pair:  item,
+                score: result[:score]
+              }
+            end
+          end
+
+          pending_items   = haystack - paired.values
+          faceted.reject do |needle, results|
+            paired.key?(needle)
+          end.each do |needle, results|
+            results.select! do |result|
+              pending_items.include?(result[:pair]) && result[:score].ratio > 0.05
+            end
+            if result = results.shift
+              paired[needle] = result
+              pending_items.delete(result[:pair])
+            end
+          end
+
+          pending_needles = needles - paired.keys
+          pending_needles.each do |needle|
+            paired[needle] = {
+              pair:  nil,
+              score: Score.new(0, needle.length)
+            }
+          end
+          paired.transform_values do |result|
+            case format
+            when Array
+              result.values_at(*format)
+            else
+              restult[format]
+            end
+          end
+        end
+
+      end
+    end
+  end
+end

data/lib/eco/data/fuzzy_match/result.rb

@@ -0,0 +1,73 @@
+module Eco
+  module Data
+    module FuzzyMatch
+      class Result < Struct.new(:match, :value, :dice, :levenshtein, :jaro_winkler, :ngrams, :words_ngrams, :chars_position)
+        ALL_METHODS = [:dice, :levenshtein, :jaro_winkler, :ngrams, :words_ngrams, :chars_position]
+
+        def dice; super&.round(3); end
+        def levenshtein; super&.round(3); end
+        def jaro_winkler; super&.round(3); end
+        def ngrams; super&.round(3); end
+        def words_ngrams; super&.round(3); end
+        def chars_position; super&.round(3); end
+
+        def average
+          values = [dice, levenshtein, jaro_winkler, ngrams, words_ngrams, chars_position]
+          (values.inject(0.0, :+) / values.length).round(3)
+        end
+
+        # TODO: print in the order of `order`
+        def print
+          msg  = "(Dice: #{dice}) (Lev Dst: #{levenshtein}) "
+          msg << "(Jaro: #{jaro_winkler}) "
+          msg << "(Ngram: #{ngrams}) (WNgrams: #{words_ngrams}) "
+          msg << "(C Pos: #{chars_position}) "
+          msg << "(Avg: #{average}) "
+          msg << "'#{value}'"
+        end
+
+        def all_threshold?(methods = order, threshold = 0.15)
+          return true unless threshold
+          [methods].flatten.compact.all? {|method| threshold?(method, threshold)}
+        end
+
+        def any_threshold?(methods = order, threshold = 0.15)
+          return true unless threshold
+          [methods].flatten.compact.any? {|method| threshold?(method, threshold)}
+        end
+
+        def threshold?(method = :dice, threshold = 0.15)
+          raise "Uknown method '#{method}'" unless self.respond_to?(method)
+          self.send(method) >= threshold
+        end
+
+        def order=(values)
+          @order = [values].flatten.compact.tap do |o|
+            o << [:words_ngrams, :dice] if o.empty?
+          end
+        end
+
+        def order
+          @order ||= [:words_ngrams, :dice]
+        end
+
+        def <=>(result)
+          compare(result)
+        end
+
+        private
+
+        def compare(other, order: self.order)
+          return 0  unless method = order.first
+          raise "Uknown method '#{method}'"  unless self.respond_to?(method) && other.respond_to?(method)
+          return -1 if self.send(method) > other.send(method)
+          return  1 if self.send(method) < other.send(method)
+          compare(other, order: order[1..-1])
+        end
+
+
+      end
+
+    end
+  end
+end

data/lib/eco/data/fuzzy_match/results.rb

@@ -0,0 +1,59 @@
+module Eco
+  module Data
+    module FuzzyMatch
+      class Results < Struct.new(:needle, :value, :raw_results)
+
+        attr_accessor :threshold
+
+        def results_with_false_positives
+          relevant_results(order: :jaro_winkler, threshold: 0.5)
+        end
+
+        def relevant_results(**options)
+          options = {order: order, threshold: threshold || 0.5}.merge(options)
+          raw_results.select do |result|
+            result.all_threshold?(options[:order], options[:threshold])
+          end.yield_self do |filtered|
+            self.class.new(needle, value, filtered).tap do |results|
+              results.order = options[:order]
+            end
+          end
+        end
+
+        # @param values[Array<Symbol>] the algorithms' results it should be ordered by
+        #   * Possible values: `:dice`, `:levenshtein`, `:jaro_winkler`, `:ngrams`, `:words_ngrams`, `:chars_position`, `:average`
+        def order=(values)
+          @order = [values].flatten.compact.tap do |o|
+            raw_results.each {|r| r.order = o}
+          end
+        end
+
+        def order
+          @order ||= [:words_ngrams, :dice]
+        end
+
+        def results
+          raw_results.sort
+        end
+
+        def print
+          msg = results.map do |result|
+            result.print
+          end.join("\n  ")
+
+          puts "'#{value}':\n  " + msg
+        end
+
+        private
+
+        def item_string(item, attr = self.method)
+          return item if !item || item.is_a?(String) || !attr
+          attr = attr.to_sym
+          return item.send(attr) if item.respond_to?(attr)
+        end
+
+      end
+
+    end
+  end
+end

data/lib/eco/data/fuzzy_match/score.rb

@@ -0,0 +1,44 @@
+module Eco
+  module Data
+    module FuzzyMatch
+      class Score < Struct.new(:score, :total)
+
+        def ratio(decimals = 6)
+          ((score || 0).to_f / (total || 1)).round(decimals)
+        end
+
+        def percent(decimals = 3)
+          (100 * ratio).round(decimals)
+        end
+
+        def increase(value = 1)
+          self.score += value
+        end
+
+        def increase_total(value)
+          self.total += value
+        end
+
+        def values_at(*keys)
+          keys.map do |key|
+            self.send(key) if self.respond_to?(key)
+          end
+        end
+
+        # Merges 2 Score instance objects
+        def merge(value)
+          Score.new(*values_at(:score, :total)).merge!(value)
+        end
+
+        def merge!(value)
+          raise "Expecting Score object. Given: #{value.class}" unless value.is_a?(Score)
+          increase(value.score)
+          increase_total(value.total)
+          self
+        end
+
+      end
+
+    end
+  end
+end