eco-helpers 2.0.18 → 2.0.24

data/lib/eco/data/fuzzy_match/chars_position_score.rb

@@ -12,8 +12,9 @@ module Eco
         def chars_position_score(str1, str2, max_distance: 3, normalized: false)
           str1, str2 = normalize_string([str1, str2]) unless normalized
           len1 = str1 && str1.length; len2 = str2 && str2.length
-          Score.new(0, len1 || 0).tap do |score|
-            next if !str1 || !str2
+          Score.new(0, 0).tap do |score|
+            next if !str2 || !str1 || str2.empty? || str1.empty?
+            score.total = len1
             next score.increase(score.total) if str1 == str2
             next if len1 < 2
             pos = 0

data/lib/eco/data/fuzzy_match/ngrams_score.rb

@@ -16,13 +16,19 @@ module Eco
 
           Score.new(0, 0).tap do |score|
             next if !str2 || !str1
-            next score.increase(score.total) if str1 == str2
-            next if str1.length < 2 || str1.length < 2
+            next score.increase_total(len1) if str2.empty? || str1.empty?
+            if str1 == str2
+              score.total = len1
+              score.increase(score.total)
+            end
+            if str1.length < 2 || str1.length < 2
+              score.increase_total(len1)
+            end
 
-            paired_words(str1, str2, normalized: true) do |needle, item|
+            pairs = paired_words(str1, str2, normalized: true) do |needle, item|
               ngrams_score(needle, item, range: range, normalized: true)
-            end.each do |sub_str1, (item, iscore)|
-              #puts "pairs '#{sub_str1}' --> '#{item}' (score: #{iscore.ratio})"
+            end.each do |sub_str1, data|
+              item, iscore = data
               score.merge!(iscore)
             end
           end
@@ -39,14 +45,17 @@ module Eco
 
           Score.new(0, len1 || 0).tap do |score|
             next if !str2 || !str1
+            next if str2.empty? || str1.empty?
+            score.total = len1
             next score.increase(score.total) if str1 == str2
             next if str1.length < 2 || str2.length < 2
 
-            grams = word_ngrams(str2, range, normalized: true)
-            next unless grams.length > 0
+            grams     = word_ngrams(str2, range, normalized: true)
+            grams_count = grams.length
+            next unless grams_count > 0
 
             if range.is_a?(Integer)
-              item_weight = score.total.to_f / grams.length
+              item_weight = score.total.to_f / grams_count
               matches     = grams.select {|res| str1.include?(gram)}.length
               score.increase(matches * item_weight)
             else
@@ -57,9 +66,9 @@ module Eco
 
               groups.each do |len, grams|
                 len_max_score  = score.total * group_weight
-                item_weight    = len_max_score / grams.length
+                item_weight    = len_max_score / grams_count
                 matches        = grams.select {|gram| str1.include?(gram)}.length
-                #pp "#{len} match: #{matches} (over #{grams.length}) || max_score: #{len_max_score} (over #{score.total})"
+                #pp "(#{len}) match: #{matches} (of #{grams.length} of total #{grams_count}) || max_score: #{len_max_score} (over #{score.total})"
                 score.increase(matches * item_weight)
               end
             end

data/lib/eco/data/fuzzy_match/pairing.rb

@@ -15,20 +15,12 @@ module Eco
         # @yieldreturn [Eco::Data::FuzzyMatch::Score] the `Score` object with the results of comparing `str1` and `str2`
         # @param str1 [String] the string of reference.
         # @param str2 [String] one of the haystack items.
-        # @param format [Symbol] determines the `values` of the returned `Hash`::
-        #   1. `:pair` for just pair
-        #   2. `:score` for just score
-        #   2. `[:pair, :score]` for `Array`
         # @normalized [Boolean] to avoid double ups in normalizing.
-        # @return [Hash] where `keys` are the **words** of `str1` and their `values`:
-        #   1. if `format` is `:pair` => the `str2` words with highest match.
-        #   2. if `format` is `:score` => the `Score` words with highest match.
-        #   3. if `format` is `[:pair, :score]` => both in an `Array`.
-        def paired_words(str1, str2, format: [:pair, :score], normalized: false)
+        # @return [Hash] where `keys` are the **words** of `str1` and their `values` a pair array of `pair` and `Score`
+        def paired_words(str1, str2, normalized: false)
           str1, str2 = normalize_string([str1, str2]) unless normalized
-          return {} if !str2 || !str1
-          return score.increase(score.total) if str1 == str2
-          return {str1 => nil} if str1.length < 2 || str1.length < 2
+          return {nil => [nil, Score.new(0, 0)]} if !str2 || !str1
+          return {str1 => [nil, Score.new(0, 0)]} if str1.length < 2 || str1.length < 2
 
           needles    = get_words(str1, normalized: true)
           haystack   = get_words(str2, normalized: true)
@@ -59,6 +51,9 @@ module Eco
               result[:score].ratio
             end.reverse
             if result = sorted.shift
+              unless result[:score].is_a?(Eco::Data::FuzzyMatch::Score)
+                raise "Parining ('#{str1}' vs '#{str2}') -> Something got sour with needle '#{result[:needle]}' and item #{item}"
+              end
               paired[result[:needle]] = {
                 pair:  item,
                 score: result[:score]
@@ -74,6 +69,9 @@ module Eco
               pending_items.include?(result[:pair]) && result[:score].ratio > 0.05
             end
             if result = results.shift
+              unless result[:score].is_a?(Eco::Data::FuzzyMatch::Score)
+                raise "Parining ('#{str1}' vs '#{str2}') -> Something got sour with needle '#{needle}' and item #{result[:pair]}"
+              end
               paired[needle] = result
               pending_items.delete(result[:pair])
             end
@@ -86,13 +84,8 @@ module Eco
               score: Score.new(0, needle.length)
             }
           end
-          paired.transform_values do |result|
-            case format
-            when Array
-              result.values_at(*format)
-            else
-              restult[format]
-            end
+          paired.each_with_object({}) do |(needle, data), out|
+            out[needle] = data.values_at(:pair, :score)
           end
         end
 

data/lib/eco/data/fuzzy_match/result.rb

@@ -1,9 +1,11 @@
 module Eco
   module Data
     module FuzzyMatch
-      class Result < Struct.new(:match, :value, :dice, :levenshtein, :jaro_winkler, :ngrams, :words_ngrams, :chars_position)
+      class Result < Struct.new(:match, :value, :needle_value, :dice, :levenshtein, :jaro_winkler, :ngrams, :words_ngrams, :chars_position)
         ALL_METHODS = [:dice, :levenshtein, :jaro_winkler, :ngrams, :words_ngrams, :chars_position]
 
+        attr_accessor :pivot
+
         def dice; super&.round(3); end
         def levenshtein; super&.round(3); end
         def jaro_winkler; super&.round(3); end
@@ -11,12 +13,24 @@ module Eco
         def words_ngrams; super&.round(3); end
         def chars_position; super&.round(3); end
 
+        #Shortcuts
+        def lev; levenshtein; end
+        def jaro; jaro_winkler; end
+        def wngrams; words_ngrams; end
+        def pos; chars_position; end
+
+        def average
+          values = [dice, levenshtein, jaro_winkler, ngrams, words_ngrams, chars_position]
+          (values.inject(0.0, :+) / values.length).round(3)
+        end
+
         # TODO: print in the order of `order`
         def print
           msg  = "(Dice: #{dice}) (Lev Dst: #{levenshtein}) "
           msg << "(Jaro: #{jaro_winkler}) "
           msg << "(Ngram: #{ngrams}) (WNgrams: #{words_ngrams}) "
           msg << "(C Pos: #{chars_position}) "
+          msg << "(Avg: #{average}) "
           msg << "'#{value}'"
         end
 
@@ -37,7 +51,7 @@ module Eco
 
         def order=(values)
           @order = [values].flatten.compact.tap do |o|
-            o = [:words_ngrams, :dice] if o.empty?
+            o << [:words_ngrams, :dice] if o.empty?
           end
         end
 
@@ -49,6 +63,12 @@ module Eco
           compare(result)
         end
 
+        def values_at(*keys)
+          keys.map do |key|
+            self.send(key) if self.respond_to?(key)
+          end
+        end
+
         private
 
         def compare(other, order: self.order)

data/lib/eco/data/fuzzy_match/results.rb

@@ -2,24 +2,48 @@ module Eco
   module Data
     module FuzzyMatch
       class Results < Struct.new(:needle, :value, :raw_results)
+        include Enumerable
+
+        attr_accessor :threshold
+
+        def empty?
+          count < 1
+        end
+
+        def each(&block)
+          return to_enum(:each) unless block
+          raw_results.each(&block)
+        end
+
+        # Merges the results of both Results object
+        def merge(res)
+          unless self.needle == res.needle
+            raise "To merge 2 Results, needle should be the same ('#{value}'). Given '#{res.value}'"
+          end
+          self.class.new(needle, value, raw_results.concat(res.raw_results))
+        end
 
         def results_with_false_positives
-          relevant_results(methods: :jaro_winkler, threshold: 0.5)
+          relevant_results(order: :jaro_winkler, threshold: 0.5)
         end
 
-        def relevant_results(methods: order, threshold: 0.5)
+        def relevant_results(**options)
+          options = {order: order, threshold: threshold || 0.5}.merge(options)
           raw_results.select do |result|
-            result.all_threshold?(methods, threshold)
+            result.all_threshold?(options[:order], options[:threshold])
           end.yield_self do |filtered|
             self.class.new(needle, value, filtered).tap do |results|
-              results.order = methods
+              results.order = options[:order]
             end
           end
         end
 
+        # @param values[Array<Symbol>] the algorithms' results it should be ordered by
+        #   * Possible values: `:dice`, `:levenshtein`, `:jaro_winkler`, `:ngrams`, `:words_ngrams`, `:chars_position`, `:average`
         def order=(values)
-          @order = [values].flatten.compact
-          raw_results.each {|r| r.order = @order}
+          @order = [values].flatten.compact.tap do |o|
+            raw_results.each {|r| r.order = o}
+          end
         end
 
         def order

data/lib/eco/data/fuzzy_match/score.rb

@@ -4,7 +4,10 @@ module Eco
       class Score < Struct.new(:score, :total)
 
         def ratio(decimals = 6)
-          ((score || 0).to_f / (total || 1)).round(decimals)
+          tot = self.total; sc = self.score
+          tot = tot && tot > 0 ? tot : 1
+          sc  = sc  && sc  > 0 ? sc  : 0
+          (sc.to_f / tot).round(decimals)
         end
 
         def percent(decimals = 3)
@@ -13,6 +16,8 @@ module Eco
 
         def increase(value = 1)
           self.score += value
+          raise "Score #{self.score} (increase: #{value}) can't be greater than total #{self.total}" if self.score > self.total
+          self.score
         end
 
         def increase_total(value)
@@ -26,14 +31,14 @@ module Eco
         end
 
         # Merges 2 Score instance objects
-        def merge(value)
-          Score.new(*values_at(:score, :total)).merge!(value)
+        def merge(scr)
+          Score.new(*values_at(:score, :total)).merge!(scr)
         end
 
-        def merge!(value)
-          raise "Expecting Score object. Given: #{value.class}" unless value.is_a?(Score)
-          increase(value.score)
-          increase_total(value.total)
+        def merge!(scr)
+          raise "Expecting Score object. Given: #{scr.class}" unless scr.is_a?(Score)
+          increase_total(scr.total)
+          increase(scr.score)
           self
         end
 

data/lib/eco/data/fuzzy_match/string_helpers.rb

@@ -17,7 +17,7 @@ module Eco
         def get_words(str, normalized: false)
           return [] unless str
           str = normalize_string(str) unless normalized
-          str.scan(/[a-zA-Z'-]+/)
+          str.scan(/[a-zA-Z'-]+/).compact
         end
 
         # Keeps the start order of the `words` and consecutive `words` together/consecutive.
@@ -63,6 +63,19 @@ module Eco
           str.tr(' ', '')
         end
 
+        # Deletes the words of `str1` and `str2` that match
+        # @return [Array<String>] pair of words.
+        def remove_matching_words(str1, str2, normalized: false)
+          unless normalized
+            str1 = normalize_string(str1)
+            str2 = normalize_string(str2)
+          end
+          return [str1, str2] if !str1 || !str2 || str1.empty? || str2.empty?
+          ws1 = get_words(str1)
+          ws2 = get_words(str2)
+          [(ws1 - ws2).join(" "), (ws2 - ws1).join(" ")]
+        end
+
       end
     end
   end

data/lib/eco/version.rb

@@ -1,3 +1,3 @@
 module Eco
-  VERSION = "2.0.18"
+  VERSION = "2.0.24"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: eco-helpers
 version: !ruby/object:Gem::Version
-  version: 2.0.18
+  version: 2.0.24
 platform: ruby
 authors:
 - Oscar Segura
@@ -230,6 +230,26 @@ dependencies:
     - - "<"
       - !ruby/object:Gem::Version
         version: '3.1'
+- !ruby/object:Gem::Dependency
+  name: hashdiff
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.0.1
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '1.1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.0.1
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '1.1'
 - !ruby/object:Gem::Dependency
   name: fuzzy_match
   requirement: !ruby/object:Gem::Requirement
@@ -290,6 +310,46 @@ dependencies:
     - - "<"
       - !ruby/object:Gem::Version
         version: '1.6'
+- !ruby/object:Gem::Dependency
+  name: roo
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.8.3
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2.9'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.8.3
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2.9'
+- !ruby/object:Gem::Dependency
+  name: roo-xls
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.2.0
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '1.3'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.2.0
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '1.3'
 description: 
 email:
 - oscar@ecoportal.co.nz
@@ -332,6 +392,7 @@ files:
 - lib/eco/api/common/people/default_parsers/policy_groups_parser.rb
 - lib/eco/api/common/people/default_parsers/select_parser.rb
 - lib/eco/api/common/people/default_parsers/send_invites_parser.rb
+- lib/eco/api/common/people/default_parsers/xls_parser.rb
 - lib/eco/api/common/people/entries.rb
 - lib/eco/api/common/people/entry_factory.rb
 - lib/eco/api/common/people/person_attribute_parser.rb
@@ -345,6 +406,8 @@ files:
 - lib/eco/api/common/session/base_session.rb
 - lib/eco/api/common/session/environment.rb
 - lib/eco/api/common/session/file_manager.rb
+- lib/eco/api/common/session/helpers.rb
+- lib/eco/api/common/session/helpers/prompt_user.rb
 - lib/eco/api/common/session/logger.rb
 - lib/eco/api/common/session/logger/cache.rb
 - lib/eco/api/common/session/logger/log.rb
@@ -397,7 +460,7 @@ files:
 - lib/eco/api/organization.rb
 - lib/eco/api/organization/login_providers.rb
 - lib/eco/api/organization/people.rb
-- lib/eco/api/organization/people_analytics.rb
+- lib/eco/api/organization/people_similarity.rb
 - lib/eco/api/organization/person_schemas.rb
 - lib/eco/api/organization/policy_groups.rb
 - lib/eco/api/organization/preferences.rb
@@ -441,6 +504,7 @@ files:
 - lib/eco/api/usecases/default_cases/analyse_people_case.rb
 - lib/eco/api/usecases/default_cases/append_usergroups_case.rb
 - lib/eco/api/usecases/default_cases/change_email_case.rb
+- lib/eco/api/usecases/default_cases/clean_unknown_tags_case.rb
 - lib/eco/api/usecases/default_cases/codes_to_tags_case.rb
 - lib/eco/api/usecases/default_cases/create_case.rb
 - lib/eco/api/usecases/default_cases/create_details_case.rb
@@ -545,7 +609,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 2.4.4
+      version: 2.5.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="

data/lib/eco/api/organization/people_analytics.rb

@@ -1,60 +0,0 @@
-module Eco
-  module API
-    module Organization
-      class PeopleAnalytics <  Eco::API::Organization::People
-        include Eco::Data::FuzzyMatch
-
-        # @!group Helpers
-
-        # @!endgroup
-
-        # @!group Searchers
-
-        # It gathers those that have the same `email`
-        # @return [Hash] where `keys` are `email`s and `values` an `Array<Person>`
-        def repeated_emails
-          init_caches
-          @by_email.select do |email, people|
-            people.count > 1
-          end
-        end
-
-        # @!endgroup
-
-        # @!group Analysers
-
-        # TODO: Sort results by `results.first.methods`
-        def similarity(**options)
-          each_with_object({}) do |person, results|
-            results[person.id] = find_all_with_score(person, **options)
-          end
-        end
-
-
-        def print_analysis(threshold)
-          similarity.each do |id, results|
-            msg = results.results.select do |result|
-              result.threshold?(threshold)
-            end.map do |result|
-              result.print
-            end.join("\n  ")
-
-            puts "'#{self[id].identify}':\n  " + msg
-          end
-        end
-        # @!endgroup
-
-        protected
-
-        def on_change
-          remove_instance_variable(@fuzzy_match)
-          super
-        end
-
-        private
-
-
-      end
-    end
-  end
-end