eco-helpers 2.0.16 → 2.0.22

data/lib/eco/data.rb

@@ -6,3 +6,4 @@ end
 require_relative 'data/crypto'
 require_relative 'data/files'
 require_relative 'data/mapper'
+require_relative 'data/fuzzy_match'

data/lib/eco/data/crypto/encryption.rb

@@ -1,7 +1,7 @@
 require 'openssl'
-require 'json'
+# 'json'
 require 'base64'
-require 'pp'
+#require 'pp'
 
 require_relative '../../cli/scripting'
 
@@ -147,7 +147,7 @@ module Eco
           return str_c
           #EncryptedData.new({content: str_c, key: key, iv: iv})
 
-          
+
         end
         def aes256_decrypt(data, key: , iv: , block_octets: BLOCK_OCTETS)
           block_bits = block_bits * 8

data/lib/eco/data/files/helpers.rb

@@ -3,11 +3,13 @@ module Eco
     module Files
       DEFAULT_TIMESTAMP_PATTERN = '%Y-%m-%dT%H%M%S'
 
-      def self.included(base)
-        base.send(:include, InstanceMethods)
-        base.extend(ClassMethods)
+      class << self
+        def included(base)
+          base.send(:include, InstanceMethods)
+          base.extend(ClassMethods)
+        end
       end
-
+      
       module InstanceMethods
 
       end

data/lib/eco/data/fuzzy_match.rb

@@ -0,0 +1,201 @@
+require 'fuzzy_match'
+require 'amatch'
+require 'jaro_winkler'
+
+require_relative 'fuzzy_match/stop_words'
+require_relative 'fuzzy_match/array_helpers'
+require_relative 'fuzzy_match/string_helpers'
+require_relative 'fuzzy_match/pairing'
+require_relative 'fuzzy_match/chars_position_score'
+require_relative 'fuzzy_match/ngrams_score'
+
+module Eco
+  module Data
+    module FuzzyMatch
+
+      class << self
+        def included(base)
+          base.send(:include, InstanceMethods)
+          base.extend(ClassMethods)
+        end
+      end
+
+      module ClassMethods
+        include ArrayHelpers
+        include StringHelpers
+        include Pairing
+        include CharsPositionScore
+        include NGramsScore
+
+        def jaro_winkler(str1, str2, **options)
+          return 0 if !str1 || !str2
+          options = {
+            ignore_case: true,
+            weight:      0.25
+          }.merge(options)
+          JaroWinkler.distance(str1, str2, **options)
+        end
+
+      end
+
+      module InstanceMethods
+        FUZZY_MATCH_OPTIONS = [
+          :identities, :groupings, :stop_words, :read,
+          :must_match_grouping, :must_match_at_least_one_word,
+          :gather_last_result, :threshold
+        ]
+
+        JARO_OPTIONS     = [:ignore_case, :weight]
+        NGRAMS_OPTIONS   = [:range]
+        POSITION_OPTIONS = [:max_distance]
+        RESULTS_OPTIONS  = [:order, :threshold]
+
+        include StopWords
+
+        attr_accessor :fuzzy_options
+
+        def fuzzy_options
+          @fuzzy_options ||= {}
+        end
+
+        def fuzzy_match(haystack_data = nil, **options)
+          if instance_variable_defined?(:@fuzzy_match) && !haystack_data
+            return @fuzzy_match if fuzzy_match_options == fuzzy_match_options(options)
+          end
+          @fuzzy_options = options
+          # make it run with a native C extension (for better performance: ~130 % increase of performance)
+          ::FuzzyMatch.engine = :amatch
+          @fuzzy_match = ::FuzzyMatch.new(haystack(haystack_data), fuzzy_match_options)
+        end
+
+        # TODO: integration for options[:unique_words] => to ensure repeated words do not bring down the score are cut by threshold
+        # @note
+        #   - When the `haystack` elements are **non** `String` objects, it excludes the needle itself from the results
+        # @param needle [String, Object] object is allowed when `fuzzy_options` includes `read:` key.
+        # @param needle_str [String, nil] the actual value of needle_str to be used.
+        # @param haystack [Enumerable] the items to find `needle` among.
+        # @return [Eco::Data::FuzzyMatch::Results]
+        def find_all_with_score(needle, needle_str: nil, haystack: nil, **options)
+          base_match    = fuzzy_match(haystack, **options)
+          match_results = base_match.find_all_with_score(needle_str || needle)
+          needle_str  ||= item_string(needle)
+          results       = match_results.each_with_object([]) do |fuzzy_results, results|
+            item, dice, lev = fuzzy_results
+            unless item == needle
+              item_str     = item_string(item)
+
+              if item_str.to_s.strip.empty? || needle_str.to_s.strip.empty?
+                dice = lev = jaro_res = ngram_res = ngram_res = wngram_res = pos_res  = 0
+              end
+
+              jaro_res     ||= jaro(needle_str, item_str)
+              ngram_res    ||= ngram(needle_str, item_str)
+              wngram_res   ||= words_ngram(needle_str, item_str)
+              pos_res      ||= position(needle_str, item_str)
+
+              results << Result.new(item, item_str, needle_str, dice, lev, jaro_res, ngram_res, wngram_res, pos_res)
+            end
+          end
+          Results.new(needle, needle_str, results).tap do |res|
+            res.order     = fuzzy_options[:order]     if fuzzy_options[:order]
+            res.threshold = fuzzy_options[:threshold] if fuzzy_options[:threshold]
+          end.relevant_results
+        end
+
+        def recalculate_results(results, needle_str: nil, **options)
+          raise "You should provide a block |needle_str, item_str, needle, item|" unless block_given?
+          new_results = results.each_with_object([]) do |result, new_results|
+            nstr, istr = yield(needle_str || results.value, result.value, results.needle, result.match)
+
+            if istr.to_s.strip.empty?
+              dice = lev = jaro_res = ngram_res = ngram_res = wngram_res = pos_res  = 1
+            elsif nstr.to_s.strip.empty?
+              unless istr = needle_str
+                dice = lev = jaro_res = ngram_res = ngram_res = wngram_res = pos_res  = 0
+              end
+            end
+
+            res          = ::FuzzyMatch.score_class.new(nstr, istr) unless dice && lev
+            dice       ||= res&.dices_coefficient_similar || 0
+            lev        ||= res&.levenshtein_similar       || 0
+            jaro_res   ||= jaro(nstr, istr)
+            ngram_res  ||= ngram(nstr, istr)
+            wngram_res ||= words_ngram(nstr, istr)
+            pos_res    ||= position(nstr, istr)
+
+            new_results << Result.new(*result.values_at(:match, :value, :needle_str), dice, lev, jaro_res, ngram_res, wngram_res, pos_res)
+          end
+          Results.new(results.needle, results.value, new_results).tap do |res|
+            res.order     = options[:order]     if options[:order]
+            res.threshold = options[:threshold] if options[:threshold]
+          end.relevant_results
+        end
+
+        private
+
+        def jaro(str1, str2)
+          options = fuzzy_options.slice(*JARO_OPTIONS)
+          self.class.jaro_winkler(str1, str2, **options)
+        end
+
+        def ngram(str1, str2)
+          options = { range: 3..5 }.merge(fuzzy_options.slice(*NGRAMS_OPTIONS))
+          self.class.ngrams_score(str1, str2, **options).ratio
+        end
+
+        def words_ngram(str1, str2)
+          options = { range: 3..7 }.merge(fuzzy_options.slice(*NGRAMS_OPTIONS))
+          self.class.words_ngrams_score(str1, str2, **options).ratio
+        end
+
+        def position(str1, str2)
+          options = fuzzy_options.slice(*POSITION_OPTIONS)
+          self.class.chars_position_score(str1, str2, **options).ratio
+        end
+
+        # @note
+        #   - When used in an `Enumerable` it will use `to_a`, or `values` if it's a `Hash`
+        # @param data [Enumerable, nil]
+        # @return [Array<Object>] the non-repeated values of `data`
+        def haystack(data = nil)
+          data = self if self.is_a?(Enumerable) && !data
+          raise "'data' should be an Enumerable. Given: #{data.class}" unless data.is_a?(Enumerable)
+          data = self.is_a?(Hash) ? self.values.flatten : to_a.flatten
+          data.uniq.compact.tap do |items|
+            if !fuzzy_read_method && found = items.find {|item| !item.is_a?(String)}
+              raise "To use non String objects as 'haystack' you should provide `read:` or `options[:read]`. Given element: #{found.class}"
+            end
+          end
+        end
+
+        def item_string(item, attr = fuzzy_read_method)
+          return item if !item || item.is_a?(String) || !attr
+          return attr.call(item) if attr.is_a?(Proc)
+          attr = attr.to_sym
+          return item.send(attr) if item.respond_to?(attr)
+        end
+
+        def fuzzy_match_options(options = nil)
+          options = fuzzy_options unless options
+          options.slice(*FUZZY_MATCH_OPTIONS).merge({
+            stop_words: PREPOSITIONS + PRONOUNS + ARTICLES
+          })
+        end
+
+        def fuzzy_read_method
+          fuzzy_match_options[:read]
+        end
+
+      end
+
+      class << self
+        include FuzzyMatch::ClassMethods
+      end
+
+    end
+  end
+end
+
+require_relative 'fuzzy_match/score'
+require_relative 'fuzzy_match/result'
+require_relative 'fuzzy_match/results'

data/lib/eco/data/fuzzy_match/array_helpers.rb

@@ -0,0 +1,75 @@
+module Eco
+  module Data
+    module FuzzyMatch
+      module ArrayHelpers
+        # Keeps the start order of the `values` and consecutive `values` together/consecutive.
+        # @param values [Array] the input array with the values.
+        # @param range [Integer, Range] determine the lenght of the generated values.
+        # @return [Array<Array<Value>>] combinations of `range` length of `values`.
+        def ngrams(values, range=2..3)
+          [].tap do |out|
+            if range.is_a?(Integer)
+              n           = range
+              values_count = values.length
+              values.each_with_index do |word, i|
+                min = i
+                max = i + (n - 1)
+                break if values_count <= max
+                out << values[min..max].join(' ')
+              end
+              out.uniq!
+            else
+              range.each {|n| out.concat(ngrams(values, n))}
+              out.uniq!
+            end
+          end
+        end
+
+        # Keeps the start order of the `values` of the input `Array` `values`.
+        # It does **not** keep consecutive `values` together (it can jump/skip items).
+        # @param values [Array] the input array with the values.
+        # @param range [Integer, Range] determine the lenght of the generated values.
+        # @return [Array<Array<Value>>] combinations of `range` length of `values`
+        def combinations(values, range=2..3)
+          if range.is_a?(Integer)
+            values.combination(range).to_a
+          else
+            range.flat_map {|size| values.combination(size).to_a}
+          end
+        end
+
+        # It includes `combinations` that break the initial order of the `Array`.
+        # It does **not** keep consecutive `values` together (it can jump/skip items).
+        # @param values [Array] the input array with the values.
+        # @param range [Integer, Range] determine the lenght of the generated values.
+        # @return [Array<Array<Value>>] permutations of `range` length of `values`
+        def permutations(values, range=2..3)
+          combinations(values, range).tap do |out|
+            range = range.is_a?(Integer)? (range..range) : range
+            out.dup.select do |item|
+              range.include?(item.length)
+            end.each do |comb|
+              comb.permutation.to_a.tap do |perms|
+                perms.each {|perm| out << perm}
+              end
+            end
+            out.uniq!
+          end
+        end
+
+        # Helper to praper facet structure
+        # @param values1 [Array] the input array with the values to have their facet against.
+        # @param values2 [Array] the input array with the values to facet against.
+        # @return [Hash] where `keys` are `values1` and `value` of each `key` all `values2`
+        def facet(values1, values2)
+          {}.tap do |out|
+            next unless values1.is_a?(Enumerable)
+            values1 = values1.is_a?(Hash) ? values1.values : values1.to_a
+            values1.each {|val| out[val] = values2.dup}
+          end
+        end
+
+      end
+    end
+  end
+end

data/lib/eco/data/fuzzy_match/chars_position_score.rb

@@ -0,0 +1,38 @@
+module Eco
+  module Data
+    module FuzzyMatch
+      module CharsPositionScore
+        # For each character in `str1`, a search is performed on `str2`.
+        # The search is deemed successful if a character is found in `str2` within `max_distance` characters of the current position.
+        # A score is kept of matching characters.
+        # @note This algorithm is best suited for matching mis-spellings.
+        # @max_distance [Integer] maximum char position distance to score.
+        # @normalized [Boolean] to avoid double ups in normalizing.
+        # @return [Score] the score object with the result.
+        def chars_position_score(str1, str2, max_distance: 3, normalized: false)
+          str1, str2 = normalize_string([str1, str2]) unless normalized
+          len1 = str1 && str1.length; len2 = str2 && str2.length
+          Score.new(0, 0).tap do |score|
+            next if !str2 || !str1 || str2.empty? || str1.empty?
+            score.total = len1
+            next score.increase(score.total) if str1 == str2
+            next if len1 < 2
+            pos = 0
+            len1.times do |i|
+              start = pos + 1
+              found = false
+              if pos = str2.index(str1[i])
+                if pos < (start + max_distance)
+                  found = true
+                  score.increase
+                end
+              end
+              pos = start unless found
+            end
+          end
+        end
+
+      end
+    end
+  end
+end

data/lib/eco/data/fuzzy_match/ngrams_score.rb

@@ -0,0 +1,82 @@
+module Eco
+  module Data
+    module FuzzyMatch
+      module NGramsScore
+        # It does the following:
+        #   1. It splits both strings into words
+        #   2. Pairs all words by best `ngrams_score` match
+        #   3. Gives `0` score to those words of `str2` that lost their pair (a word of `str1` cannot be paired twice)
+        #   4. Merges the `ngrams_score` of all the paired words of `str2` against their `str1` word pair
+        # @param range [Integer, Range] determine the lenght of the generated values for each `word`.
+        # @normalized [Boolean] to avoid double ups in normalizing.
+        # @return [Score] the score object with the result.
+        def words_ngrams_score(str1, str2, range: 3..5, normalized: false)
+          str1, str2 = normalize_string([str1, str2]) unless normalized
+          len1 = str1 && str1.length; len2 = str2 && str2.length
+
+          Score.new(0, 0).tap do |score|
+            next if !str2 || !str1
+            next score.increase_total(len1) if str2.empty? || str1.empty?
+            if str1 == str2
+              score.total = len1
+              score.increase(score.total)
+            end
+            if str1.length < 2 || str1.length < 2
+              score.increase_total(len1)
+            end
+
+            pairs = paired_words(str1, str2, normalized: true) do |needle, item|
+              ngrams_score(needle, item, range: range, normalized: true)
+            end.each do |sub_str1, data|
+              item, iscore = data
+              score.merge!(iscore)
+            end
+          end
+        end
+
+        # A score is kept of matching ngram combinations of `str2`.
+        # @note This algorithm is best suited for matching sentences, or 'firstname lastname' compared with 'lastname firstname' combinations.
+        # @param range [Integer, Range] determine the lenght of the generated values.
+        # @normalized [Boolean] to avoid double ups in normalizing.
+        # @return [Score] the score object with the result.
+        def ngrams_score(str1, str2, range: 3..5, normalized: false)
+          str1, str2 = normalize_string([str1, str2]) unless normalized
+          len1 = str1 && str1.length; len2 = str2 && str2.length
+
+          Score.new(0, len1 || 0).tap do |score|
+            next if !str2 || !str1
+            next if str2.empty? || str1.empty?
+            score.total = len1
+            next score.increase(score.total) if str1 == str2
+            next if str1.length < 2 || str2.length < 2
+
+            grams     = word_ngrams(str2, range, normalized: true)
+            grams_count = grams.length
+            next unless grams_count > 0
+
+            if range.is_a?(Integer)
+              item_weight = score.total.to_f / grams_count
+              matches     = grams.select {|res| str1.include?(gram)}.length
+              score.increase(matches * item_weight)
+            else
+              groups       = grams.group_by {|gram| gram.length}
+              sorted_lens  = groups.keys.sort.reverse
+              lens         = sorted_lens.length
+              group_weight = (1.0 / lens).round(3)
+
+              groups.each do |len, grams|
+                len_max_score  = score.total * group_weight
+                item_weight    = len_max_score / grams_count
+                matches        = grams.select {|gram| str1.include?(gram)}.length
+                #pp "(#{len}) match: #{matches} (of #{grams.length} of total #{grams_count}) || max_score: #{len_max_score} (over #{score.total})"
+                score.increase(matches * item_weight)
+              end
+            end
+
+          end
+        end
+
+      end
+    end
+  end
+end

data/lib/eco/data/fuzzy_match/pairing.rb

@@ -0,0 +1,95 @@
+module Eco
+  module Data
+    module FuzzyMatch
+      module Pairing
+
+        # Pair words using some algorithm.
+        # It does the following:
+        #   1. It splits both strings into words.
+        #   2. Pairs all words by using `block` to score the best match.
+        #   3. Gives `0` score to those words of `str2` that lost their pair (a word of `str1` cannot be paired twice).
+        #   4. Merges the `Score` of all the paired words of `str2` against their `str1` word pair.
+        # @yield [needle, item] offers a comparison algorithm between two strings.
+        # @yieldparam needle [String] the string of reference.
+        # @yieldparam item [String] one of the haystack items.
+        # @yieldreturn [Eco::Data::FuzzyMatch::Score] the `Score` object with the results of comparing `str1` and `str2`
+        # @param str1 [String] the string of reference.
+        # @param str2 [String] one of the haystack items.
+        # @normalized [Boolean] to avoid double ups in normalizing.
+        # @return [Hash] where `keys` are the **words** of `str1` and their `values` a pair array of `pair` and `Score`
+        def paired_words(str1, str2, normalized: false)
+          str1, str2 = normalize_string([str1, str2]) unless normalized
+          return {nil => [nil, Score.new(0, 0)]} if !str2 || !str1
+          return {str1 => [nil, Score.new(0, 0)]} if str1.length < 2 || str1.length < 2
+
+          needles    = get_words(str1, normalized: true)
+          haystack   = get_words(str2, normalized: true)
+
+          ranking    = {}
+          faceted    = needles.each_with_object({}) do |needle, faceted|
+            faceted[needle] = haystack.map do |item|
+              {
+                pair:  item,
+                score: yield(needle, item)
+              }.tap do |result|
+                ranking[item] ||= []
+                if result[:score].ratio > 0.05
+                  ranking[item] << ({needle: needle, score: result[:score]})
+                end
+              end
+            end.sort_by do |result|
+              result[:score].ratio
+            end.reverse
+          end
+
+          paired = {}
+          #scores = {}
+          ranking.each do |item, results|
+            sorted = results.reject do |result|
+              paired.key?(result[:needle])
+            end.sort_by do |result|
+              result[:score].ratio
+            end.reverse
+            if result = sorted.shift
+              unless result[:score].is_a?(Eco::Data::FuzzyMatch::Score)
+                raise "Parining ('#{str1}' vs '#{str2}') -> Something got sour with needle '#{result[:needle]}' and item #{item}"
+              end
+              paired[result[:needle]] = {
+                pair:  item,
+                score: result[:score]
+              }
+            end
+          end
+
+          pending_items   = haystack - paired.values
+          faceted.reject do |needle, results|
+            paired.key?(needle)
+          end.each do |needle, results|
+            results.select! do |result|
+              pending_items.include?(result[:pair]) && result[:score].ratio > 0.05
+            end
+            if result = results.shift
+              unless result[:score].is_a?(Eco::Data::FuzzyMatch::Score)
+                raise "Parining ('#{str1}' vs '#{str2}') -> Something got sour with needle '#{needle}' and item #{result[:pair]}"
+              end
+              paired[needle] = result
+              pending_items.delete(result[:pair])
+            end
+          end
+
+          pending_needles = needles - paired.keys
+          pending_needles.each do |needle|
+            paired[needle] = {
+              pair:  nil,
+              score: Score.new(0, needle.length)
+            }
+          end
+          paired.each_with_object({}) do |(needle, data), out|
+            out[needle] = data.values_at(:pair, :score)
+          end
+        end
+
+      end
+    end
+  end
+end