eco-helpers 2.0.19 → 2.0.25

data/lib/eco/data/fuzzy_match.rb

@@ -28,6 +28,7 @@ module Eco
         include NGramsScore
 
         def jaro_winkler(str1, str2, **options)
+          return 0 if !str1 || !str2
           options = {
             ignore_case: true,
             weight:      0.25
@@ -67,28 +68,67 @@ module Eco
           @fuzzy_match = ::FuzzyMatch.new(haystack(haystack_data), fuzzy_match_options)
         end
 
+        # TODO: integration for options[:unique_words] => to ensure repeated words do not bring down the score are cut by threshold
         # @note
         #   - When the `haystack` elements are **non** `String` objects, it excludes the needle itself from the results
-        # @param needle [String, Object] object is allowed when `fuzzy_options` includes `read:` key
+        # @param needle [String, Object] object is allowed when `fuzzy_options` includes `read:` key.
+        # @param needle_str [String, nil] the actual value of needle_str to be used.
+        # @param haystack [Enumerable] the items to find `needle` among.
         # @return [Eco::Data::FuzzyMatch::Results]
-        def find_all_with_score(needle, **options)
-          results = fuzzy_match(**options).find_all_with_score(needle).each_with_object([]) do |fuzzy_results, results|
+        def find_all_with_score(needle, needle_str: nil, haystack: nil, **options)
+          base_match    = fuzzy_match(haystack, **options)
+          match_results = base_match.find_all_with_score(needle_str || needle)
+          needle_str  ||= item_string(needle)
+          results       = match_results.each_with_object([]) do |fuzzy_results, results|
             item, dice, lev = fuzzy_results
             unless item == needle
-              needle_str = item_string(needle)
-              item_str   = item_string(item)
-              jaro_res   = jaro(needle_str, item_str)
-              ngram_res  = ngram(needle_str, item_str)
-              wngram_res = words_ngram(needle_str, item_str)
-              pos_res    = position(needle_str, item_str)
-
-              results << Result.new(item, item_str, dice, lev, jaro_res, ngram_res, wngram_res, pos_res)
+              item_str     = item_string(item)
+
+              if item_str.to_s.strip.empty? || needle_str.to_s.strip.empty?
+                dice = lev = jaro_res = ngram_res = ngram_res = wngram_res = pos_res  = 0
+              end
+
+              jaro_res     ||= jaro(needle_str, item_str)
+              ngram_res    ||= ngram(needle_str, item_str)
+              wngram_res   ||= words_ngram(needle_str, item_str)
+              pos_res      ||= position(needle_str, item_str)
+
+              results << Result.new(item, item_str, needle_str, dice, lev, jaro_res, ngram_res, wngram_res, pos_res)
             end
           end
-          Results.new(needle, item_string(needle), results).tap do |res|
+          Results.new(needle, needle_str, results).tap do |res|
             res.order     = fuzzy_options[:order]     if fuzzy_options[:order]
             res.threshold = fuzzy_options[:threshold] if fuzzy_options[:threshold]
+          end.relevant_results
+        end
+
+        def recalculate_results(results, needle_str: nil, **options)
+          raise "You should provide a block |needle_str, item_str, needle, item|" unless block_given?
+          new_results = results.each_with_object([]) do |result, new_results|
+            nstr, istr = yield(needle_str || results.value, result.value, results.needle, result.match)
+
+            if istr.to_s.strip.empty?
+              dice = lev = jaro_res = ngram_res = ngram_res = wngram_res = pos_res  = 1
+            elsif nstr.to_s.strip.empty?
+              unless istr = needle_str
+                dice = lev = jaro_res = ngram_res = ngram_res = wngram_res = pos_res  = 0
+              end
+            end
+
+            res          = ::FuzzyMatch.score_class.new(nstr, istr) unless dice && lev
+            dice       ||= res&.dices_coefficient_similar || 0
+            lev        ||= res&.levenshtein_similar       || 0
+            jaro_res   ||= jaro(nstr, istr)
+            ngram_res  ||= ngram(nstr, istr)
+            wngram_res ||= words_ngram(nstr, istr)
+            pos_res    ||= position(nstr, istr)
+
+            new_results << Result.new(*result.values_at(:match, :value, :needle_str), dice, lev, jaro_res, ngram_res, wngram_res, pos_res)
           end
+          Results.new(results.needle, results.value, new_results).tap do |res|
+            res.order     = options[:order]     if options[:order]
+            res.threshold = options[:threshold] if options[:threshold]
+          end.relevant_results
         end
 
         private

data/lib/eco/data/fuzzy_match/chars_position_score.rb

@@ -12,8 +12,9 @@ module Eco
         def chars_position_score(str1, str2, max_distance: 3, normalized: false)
           str1, str2 = normalize_string([str1, str2]) unless normalized
           len1 = str1 && str1.length; len2 = str2 && str2.length
-          Score.new(0, len1 || 0).tap do |score|
-            next if !str1 || !str2
+          Score.new(0, 0).tap do |score|
+            next if !str2 || !str1 || str2.empty? || str1.empty?
+            score.total = len1
             next score.increase(score.total) if str1 == str2
             next if len1 < 2
             pos = 0

data/lib/eco/data/fuzzy_match/ngrams_score.rb

@@ -16,18 +16,19 @@ module Eco
 
           Score.new(0, 0).tap do |score|
             next if !str2 || !str1
+            next score.increase_total(len1) if str2.empty? || str1.empty?
             if str1 == str2
-              score.increase_total(len1)
+              score.total = len1
               score.increase(score.total)
             end
             if str1.length < 2 || str1.length < 2
               score.increase_total(len1)
             end
 
-            paired_words(str1, str2, normalized: true) do |needle, item|
+            pairs = paired_words(str1, str2, normalized: true) do |needle, item|
               ngrams_score(needle, item, range: range, normalized: true)
-            end.each do |sub_str1, (item, iscore)|
-              #puts "pairs '#{sub_str1}' --> '#{item}' (score: #{iscore.ratio})"
+            end.each do |sub_str1, data|
+              item, iscore = data
               score.merge!(iscore)
             end
           end
@@ -44,14 +45,17 @@ module Eco
 
           Score.new(0, len1 || 0).tap do |score|
             next if !str2 || !str1
+            next if str2.empty? || str1.empty?
+            score.total = len1
             next score.increase(score.total) if str1 == str2
             next if str1.length < 2 || str2.length < 2
 
-            grams = word_ngrams(str2, range, normalized: true)
-            next unless grams.length > 0
+            grams     = word_ngrams(str2, range, normalized: true)
+            grams_count = grams.length
+            next unless grams_count > 0
 
             if range.is_a?(Integer)
-              item_weight = score.total.to_f / grams.length
+              item_weight = score.total.to_f / grams_count
               matches     = grams.select {|res| str1.include?(gram)}.length
               score.increase(matches * item_weight)
             else
@@ -62,9 +66,9 @@ module Eco
 
               groups.each do |len, grams|
                 len_max_score  = score.total * group_weight
-                item_weight    = len_max_score / grams.length
+                item_weight    = len_max_score / grams_count
                 matches        = grams.select {|gram| str1.include?(gram)}.length
-                #pp "#{len} match: #{matches} (over #{grams.length}) || max_score: #{len_max_score} (over #{score.total})"
+                #pp "(#{len}) match: #{matches} (of #{grams.length} of total #{grams_count}) || max_score: #{len_max_score} (over #{score.total})"
                 score.increase(matches * item_weight)
               end
             end

data/lib/eco/data/fuzzy_match/pairing.rb

@@ -15,19 +15,12 @@ module Eco
         # @yieldreturn [Eco::Data::FuzzyMatch::Score] the `Score` object with the results of comparing `str1` and `str2`
         # @param str1 [String] the string of reference.
         # @param str2 [String] one of the haystack items.
-        # @param format [Symbol] determines the `values` of the returned `Hash`::
-        #   1. `:pair` for just pair
-        #   2. `:score` for just score
-        #   2. `[:pair, :score]` for `Array`
         # @normalized [Boolean] to avoid double ups in normalizing.
-        # @return [Hash] where `keys` are the **words** of `str1` and their `values`:
-        #   1. if `format` is `:pair` => the `str2` words with highest match.
-        #   2. if `format` is `:score` => the `Score` words with highest match.
-        #   3. if `format` is `[:pair, :score]` => both in an `Array`.
-        def paired_words(str1, str2, format: [:pair, :score], normalized: false)
+        # @return [Hash] where `keys` are the **words** of `str1` and their `values` a pair array of `pair` and `Score`
+        def paired_words(str1, str2, normalized: false)
           str1, str2 = normalize_string([str1, str2]) unless normalized
-          return {} if !str2 || !str1
-          return {str1 => nil} if str1.length < 2 || str1.length < 2
+          return {nil => [nil, Score.new(0, 0)]} if !str2 || !str1
+          return {str1 => [nil, Score.new(0, 0)]} if str1.length < 2 || str1.length < 2
 
           needles    = get_words(str1, normalized: true)
           haystack   = get_words(str2, normalized: true)
@@ -58,6 +51,9 @@ module Eco
               result[:score].ratio
             end.reverse
             if result = sorted.shift
+              unless result[:score].is_a?(Eco::Data::FuzzyMatch::Score)
+                raise "Parining ('#{str1}' vs '#{str2}') -> Something got sour with needle '#{result[:needle]}' and item #{item}"
+              end
               paired[result[:needle]] = {
                 pair:  item,
                 score: result[:score]
@@ -73,6 +69,9 @@ module Eco
               pending_items.include?(result[:pair]) && result[:score].ratio > 0.05
             end
             if result = results.shift
+              unless result[:score].is_a?(Eco::Data::FuzzyMatch::Score)
+                raise "Parining ('#{str1}' vs '#{str2}') -> Something got sour with needle '#{needle}' and item #{result[:pair]}"
+              end
               paired[needle] = result
               pending_items.delete(result[:pair])
             end
@@ -85,13 +84,8 @@ module Eco
               score: Score.new(0, needle.length)
             }
           end
-          paired.transform_values do |result|
-            case format
-            when Array
-              result.values_at(*format)
-            else
-              restult[format]
-            end
+          paired.each_with_object({}) do |(needle, data), out|
+            out[needle] = data.values_at(:pair, :score)
           end
         end
 

data/lib/eco/data/fuzzy_match/result.rb

@@ -1,9 +1,11 @@
 module Eco
   module Data
     module FuzzyMatch
-      class Result < Struct.new(:match, :value, :dice, :levenshtein, :jaro_winkler, :ngrams, :words_ngrams, :chars_position)
+      class Result < Struct.new(:match, :value, :needle_value, :dice, :levenshtein, :jaro_winkler, :ngrams, :words_ngrams, :chars_position)
         ALL_METHODS = [:dice, :levenshtein, :jaro_winkler, :ngrams, :words_ngrams, :chars_position]
 
+        attr_accessor :pivot
+
         def dice; super&.round(3); end
         def levenshtein; super&.round(3); end
         def jaro_winkler; super&.round(3); end
@@ -11,6 +13,12 @@ module Eco
         def words_ngrams; super&.round(3); end
         def chars_position; super&.round(3); end
 
+        #Shortcuts
+        def lev; levenshtein; end
+        def jaro; jaro_winkler; end
+        def wngrams; words_ngrams; end
+        def pos; chars_position; end
+
         def average
           values = [dice, levenshtein, jaro_winkler, ngrams, words_ngrams, chars_position]
           (values.inject(0.0, :+) / values.length).round(3)
@@ -55,6 +63,12 @@ module Eco
           compare(result)
         end
 
+        def values_at(*keys)
+          keys.map do |key|
+            self.send(key) if self.respond_to?(key)
+          end
+        end
+
         private
 
         def compare(other, order: self.order)

data/lib/eco/data/fuzzy_match/results.rb

@@ -2,9 +2,27 @@ module Eco
   module Data
     module FuzzyMatch
       class Results < Struct.new(:needle, :value, :raw_results)
+        include Enumerable
 
         attr_accessor :threshold
 
+        def empty?
+          count < 1
+        end
+
+        def each(&block)
+          return to_enum(:each) unless block
+          raw_results.each(&block)
+        end
+
+        # Merges the results of both Results object
+        def merge(res)
+          unless self.needle == res.needle
+            raise "To merge 2 Results, needle should be the same ('#{value}'). Given '#{res.value}'"
+          end
+          self.class.new(needle, value, raw_results.concat(res.raw_results))
+        end
+
         def results_with_false_positives
           relevant_results(order: :jaro_winkler, threshold: 0.5)
         end

data/lib/eco/data/fuzzy_match/score.rb

@@ -4,7 +4,10 @@ module Eco
       class Score < Struct.new(:score, :total)
 
         def ratio(decimals = 6)
-          ((score || 0).to_f / (total || 1)).round(decimals)
+          tot = self.total; sc = self.score
+          tot = tot && tot > 0 ? tot : 1
+          sc  = sc  && sc  > 0 ? sc  : 0
+          (sc.to_f / tot).round(decimals)
         end
 
         def percent(decimals = 3)
@@ -13,6 +16,8 @@ module Eco
 
         def increase(value = 1)
           self.score += value
+          raise "Score #{self.score} (increase: #{value}) can't be greater than total #{self.total}" if self.score > self.total
+          self.score
         end
 
         def increase_total(value)
@@ -26,14 +31,14 @@ module Eco
         end
 
         # Merges 2 Score instance objects
-        def merge(value)
-          Score.new(*values_at(:score, :total)).merge!(value)
+        def merge(scr)
+          Score.new(*values_at(:score, :total)).merge!(scr)
         end
 
-        def merge!(value)
-          raise "Expecting Score object. Given: #{value.class}" unless value.is_a?(Score)
-          increase(value.score)
-          increase_total(value.total)
+        def merge!(scr)
+          raise "Expecting Score object. Given: #{scr.class}" unless scr.is_a?(Score)
+          increase_total(scr.total)
+          increase(scr.score)
           self
         end
 

data/lib/eco/data/fuzzy_match/string_helpers.rb

@@ -17,7 +17,7 @@ module Eco
         def get_words(str, normalized: false)
           return [] unless str
           str = normalize_string(str) unless normalized
-          str.scan(/[a-zA-Z'-]+/)
+          str.scan(/[a-zA-Z'-]+/).compact
         end
 
         # Keeps the start order of the `words` and consecutive `words` together/consecutive.
@@ -63,6 +63,19 @@ module Eco
           str.tr(' ', '')
         end
 
+        # Deletes the words of `str1` and `str2` that match
+        # @return [Array<String>] pair of words.
+        def remove_matching_words(str1, str2, normalized: false)
+          unless normalized
+            str1 = normalize_string(str1)
+            str2 = normalize_string(str2)
+          end
+          return [str1, str2] if !str1 || !str2 || str1.empty? || str2.empty?
+          ws1 = get_words(str1)
+          ws2 = get_words(str2)
+          [(ws1 - ws2).join(" "), (ws2 - ws1).join(" ")]
+        end
+
       end
     end
   end

data/lib/eco/language/models/collection.rb

@@ -134,8 +134,11 @@ module Eco
           to_a.group_by(&block) if block
         end
 
-        def to_h(attr)
-          return {} if !attr
+        # By a specific `attr` or a block
+        # @note either one or the other should be present
+        def to_h(attr, &block)
+          return to_a.group_by(&block) if block
+          raise "And attr or a block are required. Given attr: #{attr}" unless attr
           to_a.group_by { |object| object.method(attr).call }
         end
         # @!endgroup

data/lib/eco/version.rb

@@ -1,3 +1,3 @@
 module Eco
-  VERSION = "2.0.19"
+  VERSION = "2.0.25"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: eco-helpers
 version: !ruby/object:Gem::Version
-  version: 2.0.19
+  version: 2.0.25
 platform: ruby
 authors:
 - Oscar Segura
@@ -230,6 +230,26 @@ dependencies:
     - - "<"
       - !ruby/object:Gem::Version
         version: '3.1'
+- !ruby/object:Gem::Dependency
+  name: hashdiff
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.0.1
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '1.1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.0.1
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '1.1'
 - !ruby/object:Gem::Dependency
   name: fuzzy_match
   requirement: !ruby/object:Gem::Requirement
@@ -290,6 +310,46 @@ dependencies:
     - - "<"
       - !ruby/object:Gem::Version
         version: '1.6'
+- !ruby/object:Gem::Dependency
+  name: roo
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.8.3
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2.9'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.8.3
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2.9'
+- !ruby/object:Gem::Dependency
+  name: roo-xls
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.2.0
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '1.3'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.2.0
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '1.3'
 description: 
 email:
 - oscar@ecoportal.co.nz
@@ -332,6 +392,7 @@ files:
 - lib/eco/api/common/people/default_parsers/policy_groups_parser.rb
 - lib/eco/api/common/people/default_parsers/select_parser.rb
 - lib/eco/api/common/people/default_parsers/send_invites_parser.rb
+- lib/eco/api/common/people/default_parsers/xls_parser.rb
 - lib/eco/api/common/people/entries.rb
 - lib/eco/api/common/people/entry_factory.rb
 - lib/eco/api/common/people/person_attribute_parser.rb
@@ -443,6 +504,7 @@ files:
 - lib/eco/api/usecases/default_cases/analyse_people_case.rb
 - lib/eco/api/usecases/default_cases/append_usergroups_case.rb
 - lib/eco/api/usecases/default_cases/change_email_case.rb
+- lib/eco/api/usecases/default_cases/clean_unknown_tags_case.rb
 - lib/eco/api/usecases/default_cases/codes_to_tags_case.rb
 - lib/eco/api/usecases/default_cases/create_case.rb
 - lib/eco/api/usecases/default_cases/create_details_case.rb
@@ -547,7 +609,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 2.4.4
+      version: 2.5.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="