eco-helpers 2.0.13 → 2.0.18

data/lib/eco/data/files/helpers.rb

@@ -3,11 +3,13 @@ module Eco
     module Files
       DEFAULT_TIMESTAMP_PATTERN = '%Y-%m-%dT%H%M%S'
 
-      def self.included(base)
-        base.send(:include, InstanceMethods)
-        base.extend(ClassMethods)
+      class << self
+        def included(base)
+          base.send(:include, InstanceMethods)
+          base.extend(ClassMethods)
+        end
       end
-
+      
       module InstanceMethods
 
       end

data/lib/eco/data/fuzzy_match.rb

@@ -0,0 +1,119 @@
+require 'fuzzy_match'
+require 'amatch'
+require 'jaro_winkler'
+
+require_relative 'fuzzy_match/stop_words'
+require_relative 'fuzzy_match/array_helpers'
+require_relative 'fuzzy_match/string_helpers'
+require_relative 'fuzzy_match/pairing'
+require_relative 'fuzzy_match/chars_position_score'
+require_relative 'fuzzy_match/ngrams_score'
+
+module Eco
+  module Data
+    module FuzzyMatch
+
+      class << self
+        def included(base)
+          base.send(:include, InstanceMethods)
+          base.extend(ClassMethods)
+        end
+      end
+
+      module ClassMethods
+        include ArrayHelpers
+        include StringHelpers
+        include Pairing
+        include CharsPositionScore
+        include NGramsScore
+
+        def jaro_winkler(str1, str2)
+          options = {
+            ignore_case: true,
+            weight:      0.25
+          }
+          JaroWinkler.distance(str1, str2, **options)
+        end
+
+      end
+
+      module InstanceMethods
+        include StopWords
+
+        attr_accessor :fuzzy_options
+
+        def fuzzy_options
+          @fuzzy_options ||= {}
+        end
+
+        def fuzzy_match(haystack = nil, **options)
+          return @fuzzy_match if instance_variable_defined?(:@fuzzy_match)
+          @fuzzy_options = options.merge({
+            stop_words: PREPOSITIONS + PRONOUNS + ARTICLES
+          })
+          # make it run with a native C extension (for better performance: ~130 % increase of performance)
+          ::FuzzyMatch.engine = :amatch
+          haystack = obtain_haystack(haystack).tap do |items|
+            if !fuzzy_read_method && found = items.find {|item| !item.is_a?(String)}
+              raise "To use non String objects as 'haystack' you should provide `read:` or `options[:read]`. Given element: #{found.class}"
+            end
+          end
+          @fuzzy_match = ::FuzzyMatch.new(haystack, fuzzy_options)
+        end
+
+        # @note
+        #   - When the `haystack` elements are **non** `String` objects, it excludes the needle itself from the results
+        # @param needle [String, Object] object is allowed when `fuzzy_options` includes `read:` key
+        # @return [Eco::Data::FuzzyMatch::Results]
+        def find_all_with_score(needle, **options)
+          results = fuzzy_match(**options).find_all_with_score(needle).each_with_object([]) do |fuzzy_results, results|
+            item, dice, lev = fuzzy_results
+            unless item == needle
+              needle_str = item_string(needle)
+              item_str   = item_string(item)
+              jaro_res   = self.class.jaro_winkler(needle_str, item_str)
+              ngram_res  = self.class.ngrams_score(needle_str, item_str, range: 3..5).ratio
+              wngram_res = self.class.words_ngrams_score(needle_str, item_str, range: 3..7).ratio
+              pos_res    = self.class.chars_position_score(needle_str, item_str).ratio
+              results << Result.new(item, item_str, dice, lev, jaro_res, ngram_res, wngram_res, pos_res)
+            end
+          end
+          Results.new(needle, item_string(needle), results)
+        end
+
+        private
+
+        # @note
+        #   - When used in an `Enumerable` it will use `to_a`, or `values` if it's a `Hash`
+        # @param data [Enumerable, nil]
+        # @return [Array<Object>] the non-repeated values of `data`
+        def obtain_haystack(data = nil)
+          data = self if self.is_a?(Enumerable) && !data
+          raise "'data' should be an Enumerable. Given: #{data.class}" unless data.is_a?(Enumerable)
+          data = self.is_a?(Hash) ? self.values.flatten : to_a.flatten
+          data.uniq.compact
+        end
+
+        def item_string(item, attr = fuzzy_read_method)
+          return item if !item || item.is_a?(String) || !attr
+          attr = attr.to_sym
+          return item.send(attr) if item.respond_to?(attr)
+        end
+
+        def fuzzy_read_method
+          fuzzy_options[:read]
+        end
+
+      end
+
+      class << self
+        include FuzzyMatch::ClassMethods
+      end
+
+    end
+  end
+end
+
+require_relative 'fuzzy_match/score'
+require_relative 'fuzzy_match/result'
+require_relative 'fuzzy_match/results'

data/lib/eco/data/fuzzy_match/array_helpers.rb

@@ -0,0 +1,75 @@
+module Eco
+  module Data
+    module FuzzyMatch
+      module ArrayHelpers
+        # Keeps the start order of the `values` and consecutive `values` together/consecutive.
+        # @param values [Array] the input array with the values.
+        # @param range [Integer, Range] determine the lenght of the generated values.
+        # @return [Array<Array<Value>>] combinations of `range` length of `values`.
+        def ngrams(values, range=2..3)
+          [].tap do |out|
+            if range.is_a?(Integer)
+              n           = range
+              values_count = values.length
+              values.each_with_index do |word, i|
+                min = i
+                max = i + (n - 1)
+                break if values_count <= max
+                out << values[min..max].join(' ')
+              end
+              out.uniq!
+            else
+              range.each {|n| out.concat(ngrams(values, n))}
+              out.uniq!
+            end
+          end
+        end
+
+        # Keeps the start order of the `values` of the input `Array` `values`.
+        # It does **not** keep consecutive `values` together (it can jump/skip items).
+        # @param values [Array] the input array with the values.
+        # @param range [Integer, Range] determine the lenght of the generated values.
+        # @return [Array<Array<Value>>] combinations of `range` length of `values`
+        def combinations(values, range=2..3)
+          if range.is_a?(Integer)
+            values.combination(range).to_a
+          else
+            range.flat_map {|size| values.combination(size).to_a}
+          end
+        end
+
+        # It includes `combinations` that break the initial order of the `Array`.
+        # It does **not** keep consecutive `values` together (it can jump/skip items).
+        # @param values [Array] the input array with the values.
+        # @param range [Integer, Range] determine the lenght of the generated values.
+        # @return [Array<Array<Value>>] permutations of `range` length of `values`
+        def permutations(values, range=2..3)
+          combinations(values, range).tap do |out|
+            range = range.is_a?(Integer)? (range..range) : range
+            out.dup.select do |item|
+              range.include?(item.length)
+            end.each do |comb|
+              comb.permutation.to_a.tap do |perms|
+                perms.each {|perm| out << perm}
+              end
+            end
+            out.uniq!
+          end
+        end
+
+        # Helper to praper facet structure
+        # @param values1 [Array] the input array with the values to have their facet against.
+        # @param values2 [Array] the input array with the values to facet against.
+        # @return [Hash] where `keys` are `values1` and `value` of each `key` all `values2`
+        def facet(values1, values2)
+          {}.tap do |out|
+            next unless values1.is_a?(Enumerable)
+            values1 = values1.is_a?(Hash) ? values1.values : values1.to_a
+            values1.each {|val| out[val] = values2.dup}
+          end
+        end
+
+      end
+    end
+  end
+end

data/lib/eco/data/fuzzy_match/chars_position_score.rb

@@ -0,0 +1,37 @@
+module Eco
+  module Data
+    module FuzzyMatch
+      module CharsPositionScore
+        # For each character in `str1`, a search is performed on `str2`.
+        # The search is deemed successful if a character is found in `str2` within `max_distance` characters of the current position.
+        # A score is kept of matching characters.
+        # @note This algorithm is best suited for matching mis-spellings.
+        # @max_distance [Integer] maximum char position distance to score.
+        # @normalized [Boolean] to avoid double ups in normalizing.
+        # @return [Score] the score object with the result.
+        def chars_position_score(str1, str2, max_distance: 3, normalized: false)
+          str1, str2 = normalize_string([str1, str2]) unless normalized
+          len1 = str1 && str1.length; len2 = str2 && str2.length
+          Score.new(0, len1 || 0).tap do |score|
+            next if !str1 || !str2
+            next score.increase(score.total) if str1 == str2
+            next if len1 < 2
+            pos = 0
+            len1.times do |i|
+              start = pos + 1
+              found = false
+              if pos = str2.index(str1[i])
+                if pos < (start + max_distance)
+                  found = true
+                  score.increase
+                end
+              end
+              pos = start unless found
+            end
+          end
+        end
+
+      end
+    end
+  end
+end

data/lib/eco/data/fuzzy_match/ngrams_score.rb

@@ -0,0 +1,73 @@
+module Eco
+  module Data
+    module FuzzyMatch
+      module NGramsScore
+        # It does the following:
+        #   1. It splits both strings into words
+        #   2. Pairs all words by best `ngrams_score` match
+        #   3. Gives `0` score to those words of `str2` that lost their pair (a word of `str1` cannot be paired twice)
+        #   4. Merges the `ngrams_score` of all the paired words of `str2` against their `str1` word pair
+        # @param range [Integer, Range] determine the lenght of the generated values for each `word`.
+        # @normalized [Boolean] to avoid double ups in normalizing.
+        # @return [Score] the score object with the result.
+        def words_ngrams_score(str1, str2, range: 3..5, normalized: false)
+          str1, str2 = normalize_string([str1, str2]) unless normalized
+          len1 = str1 && str1.length; len2 = str2 && str2.length
+
+          Score.new(0, 0).tap do |score|
+            next if !str2 || !str1
+            next score.increase(score.total) if str1 == str2
+            next if str1.length < 2 || str1.length < 2
+
+            paired_words(str1, str2, normalized: true) do |needle, item|
+              ngrams_score(needle, item, range: range, normalized: true)
+            end.each do |sub_str1, (item, iscore)|
+              #puts "pairs '#{sub_str1}' --> '#{item}' (score: #{iscore.ratio})"
+              score.merge!(iscore)
+            end
+          end
+        end
+
+        # A score is kept of matching ngram combinations of `str2`.
+        # @note This algorithm is best suited for matching sentences, or 'firstname lastname' compared with 'lastname firstname' combinations.
+        # @param range [Integer, Range] determine the lenght of the generated values.
+        # @normalized [Boolean] to avoid double ups in normalizing.
+        # @return [Score] the score object with the result.
+        def ngrams_score(str1, str2, range: 3..5, normalized: false)
+          str1, str2 = normalize_string([str1, str2]) unless normalized
+          len1 = str1 && str1.length; len2 = str2 && str2.length
+
+          Score.new(0, len1 || 0).tap do |score|
+            next if !str2 || !str1
+            next score.increase(score.total) if str1 == str2
+            next if str1.length < 2 || str2.length < 2
+
+            grams = word_ngrams(str2, range, normalized: true)
+            next unless grams.length > 0
+
+            if range.is_a?(Integer)
+              item_weight = score.total.to_f / grams.length
+              matches     = grams.select {|res| str1.include?(gram)}.length
+              score.increase(matches * item_weight)
+            else
+              groups       = grams.group_by {|gram| gram.length}
+              sorted_lens  = groups.keys.sort.reverse
+              lens         = sorted_lens.length
+              group_weight = (1.0 / lens).round(3)
+
+              groups.each do |len, grams|
+                len_max_score  = score.total * group_weight
+                item_weight    = len_max_score / grams.length
+                matches        = grams.select {|gram| str1.include?(gram)}.length
+                #pp "#{len} match: #{matches} (over #{grams.length}) || max_score: #{len_max_score} (over #{score.total})"
+                score.increase(matches * item_weight)
+              end
+            end
+
+          end
+        end
+
+      end
+    end
+  end
+end

data/lib/eco/data/fuzzy_match/pairing.rb

@@ -0,0 +1,102 @@
+module Eco
+  module Data
+    module FuzzyMatch
+      module Pairing
+
+        # Pair words using some algorithm.
+        # It does the following:
+        #   1. It splits both strings into words.
+        #   2. Pairs all words by using `block` to score the best match.
+        #   3. Gives `0` score to those words of `str2` that lost their pair (a word of `str1` cannot be paired twice).
+        #   4. Merges the `Score` of all the paired words of `str2` against their `str1` word pair.
+        # @yield [needle, item] offers a comparison algorithm between two strings.
+        # @yieldparam needle [String] the string of reference.
+        # @yieldparam item [String] one of the haystack items.
+        # @yieldreturn [Eco::Data::FuzzyMatch::Score] the `Score` object with the results of comparing `str1` and `str2`
+        # @param str1 [String] the string of reference.
+        # @param str2 [String] one of the haystack items.
+        # @param format [Symbol] determines the `values` of the returned `Hash`::
+        #   1. `:pair` for just pair
+        #   2. `:score` for just score
+        #   2. `[:pair, :score]` for `Array`
+        # @normalized [Boolean] to avoid double ups in normalizing.
+        # @return [Hash] where `keys` are the **words** of `str1` and their `values`:
+        #   1. if `format` is `:pair` => the `str2` words with highest match.
+        #   2. if `format` is `:score` => the `Score` words with highest match.
+        #   3. if `format` is `[:pair, :score]` => both in an `Array`.
+        def paired_words(str1, str2, format: [:pair, :score], normalized: false)
+          str1, str2 = normalize_string([str1, str2]) unless normalized
+          return {} if !str2 || !str1
+          return score.increase(score.total) if str1 == str2
+          return {str1 => nil} if str1.length < 2 || str1.length < 2
+
+          needles    = get_words(str1, normalized: true)
+          haystack   = get_words(str2, normalized: true)
+
+          ranking    = {}
+          faceted    = needles.each_with_object({}) do |needle, faceted|
+            faceted[needle] = haystack.map do |item|
+              {
+                pair:  item,
+                score: yield(needle, item)
+              }.tap do |result|
+                ranking[item] ||= []
+                if result[:score].ratio > 0.05
+                  ranking[item] << ({needle: needle, score: result[:score]})
+                end
+              end
+            end.sort_by do |result|
+              result[:score].ratio
+            end.reverse
+          end
+
+          paired = {}
+          #scores = {}
+          ranking.each do |item, results|
+            sorted = results.reject do |result|
+              paired.key?(result[:needle])
+            end.sort_by do |result|
+              result[:score].ratio
+            end.reverse
+            if result = sorted.shift
+              paired[result[:needle]] = {
+                pair:  item,
+                score: result[:score]
+              }
+            end
+          end
+
+          pending_items   = haystack - paired.values
+          faceted.reject do |needle, results|
+            paired.key?(needle)
+          end.each do |needle, results|
+            results.select! do |result|
+              pending_items.include?(result[:pair]) && result[:score].ratio > 0.05
+            end
+            if result = results.shift
+              paired[needle] = result
+              pending_items.delete(result[:pair])
+            end
+          end
+
+          pending_needles = needles - paired.keys
+          pending_needles.each do |needle|
+            paired[needle] = {
+              pair:  nil,
+              score: Score.new(0, needle.length)
+            }
+          end
+          paired.transform_values do |result|
+            case format
+            when Array
+              result.values_at(*format)
+            else
+              restult[format]
+            end
+          end
+        end
+
+      end
+    end
+  end
+end

data/lib/eco/data/fuzzy_match/result.rb

@@ -0,0 +1,67 @@
+module Eco
+  module Data
+    module FuzzyMatch
+      class Result < Struct.new(:match, :value, :dice, :levenshtein, :jaro_winkler, :ngrams, :words_ngrams, :chars_position)
+        ALL_METHODS = [:dice, :levenshtein, :jaro_winkler, :ngrams, :words_ngrams, :chars_position]
+
+        def dice; super&.round(3); end
+        def levenshtein; super&.round(3); end
+        def jaro_winkler; super&.round(3); end
+        def ngrams; super&.round(3); end
+        def words_ngrams; super&.round(3); end
+        def chars_position; super&.round(3); end
+
+        # TODO: print in the order of `order`
+        def print
+          msg  = "(Dice: #{dice}) (Lev Dst: #{levenshtein}) "
+          msg << "(Jaro: #{jaro_winkler}) "
+          msg << "(Ngram: #{ngrams}) (WNgrams: #{words_ngrams}) "
+          msg << "(C Pos: #{chars_position}) "
+          msg << "'#{value}'"
+        end
+
+        def all_threshold?(methods = order, threshold = 0.15)
+          return true unless threshold
+          [methods].flatten.compact.all? {|method| threshold?(method, threshold)}
+        end
+
+        def any_threshold?(methods = order, threshold = 0.15)
+          return true unless threshold
+          [methods].flatten.compact.any? {|method| threshold?(method, threshold)}
+        end
+
+        def threshold?(method = :dice, threshold = 0.15)
+          raise "Uknown method '#{method}'" unless self.respond_to?(method)
+          self.send(method) >= threshold
+        end
+
+        def order=(values)
+          @order = [values].flatten.compact.tap do |o|
+            o = [:words_ngrams, :dice] if o.empty?
+          end
+        end
+
+        def order
+          @order ||= [:words_ngrams, :dice]
+        end
+
+        def <=>(result)
+          compare(result)
+        end
+
+        private
+
+        def compare(other, order: self.order)
+          return 0  unless method = order.first
+          raise "Uknown method '#{method}'"  unless self.respond_to?(method) && other.respond_to?(method)
+          return -1 if self.send(method) > other.send(method)
+          return  1 if self.send(method) < other.send(method)
+          compare(other, order: order[1..-1])
+        end
+
+
+      end
+
+    end
+  end
+end