eco-helpers 2.0.16 → 2.0.22

data/lib/eco/api/microcases/with_each_present.rb

@@ -15,8 +15,10 @@ module Eco
       def with_each_present(entries, people, options, log_starter: false)
         found = []
         micro.with_each(entries, people, options) do |entry, person|
-          if person.new? && log_starter
-            session.logger.error("This person does not exist: #{entry.to_s(:identify)}")
+          if person.new?
+            if log_starter
+              session.logger.error("This person does not exist: #{entry.to_s(:identify)}")
+            end
             next
           end
           found << person

data/lib/eco/api/microcases/with_each_starter.rb

@@ -15,8 +15,10 @@ module Eco
       def with_each_starter(entries, people, options, log_present: false)
         starters = []
         micro.with_each(entries, people, options) do |entry, person|
-          if !person.new? && log_present
-            session.logger.error("This person (id: '#{person.id}') already exists: #{entry.to_s(:identify)}")
+          if !person.new?
+            if log_present
+              session.logger.error("This person (id: '#{person.id}') already exists: #{entry.to_s(:identify)}")
+            end
             next
           end
           starters << person

data/lib/eco/api/organization.rb

@@ -9,6 +9,7 @@ require_relative 'organization/tag_tree'
 require_relative 'organization/presets_factory'
 require_relative 'organization/preferences'
 require_relative 'organization/people'
+require_relative 'organization/people_similarity'
 require_relative 'organization/person_schemas'
 require_relative 'organization/policy_groups'
 require_relative 'organization/login_providers'

data/lib/eco/api/organization/people.rb

@@ -2,6 +2,43 @@ module Eco
   module API
     module Organization
       class People <  Eco::Language::Models::Collection
+        # Error class that allows to handle cases where multiple people were found for the same criterion.
+        # @note its main purpose to prevent the creation of duplicates or override information between different people.
+        class MultipleSearchResults < StandardError
+          attr_reader :candidates, :property
+          # @param msg [String] the basic message error.
+          # @param candiates [Array<Person>] the people that match the same search criterion.
+          # @param property [String] the property of the person model that triggered the error (base of the search criterion).
+          def initialize(msg, candidates: [], property: "email")
+            @candidates = candidates
+            @property   = property
+            super(msg + " " + candidates_summary)
+          end
+
+          # @param with_index [Boolean] to add an index to each candidate description.
+          # @return [Array<String>] the `candidates` identified
+          def identify_candidates(with_index: false)
+            candidates.map.each_with_index do |person, i|
+              index = with_index     ? "#{i}. " : ""
+              msg   = person.account ? (person.account_added? ? "(new user)" : "(user)") :  "(no account)"
+              "#{index}#{msg} #{person.identify}"
+            end
+          end
+
+          # @return [Person] the `candidate` in the `index` position
+          def candidate(index)
+            candidates[index]
+          end
+
+          private
+
+          def candidates_summary
+            lines = ["The following people have the same '#{property}':"]
+            lines.concat(identify_candidates(with_index: true)).join("\n  ")
+          end
+
+        end
+
         # build the shortcuts of Collection
         attr_presence :account, :details
         attr_collection :id, :external_id, :email, :name, :supervisor_id
@@ -78,34 +115,36 @@ module Eco
         # @!group Searchers
 
         # It searches a person using the parameters given.
+        # @note This is how the search function actually works:
+        #   1. if eP `id` is given, returns the person (if found), otherwise...
+        #   2. if `external_id` is given, returns the person (if found), otherwise...
+        #   3. if `strict` is `false` and `email` is given:
+        #     - if there is only 1 person with that email, returns that person, otherwise...
+        #     - if found but, there are many candidates, it raises MultipleSearchResults error
+        #     - if person `external_id` matches `email`, returns that person
+        # @raise MultipleSearchResults if there are multiple people with the same `email`
+        #   and there's no other criteria to find the person. It only gets to this point if
+        #   `external_id` was **not** provided and we are **not** in 'strict' search mode.
+        #   However, it could be we were in `strict` mode and `external_id` was not provided.
         # @param id [String] the `internal id` of the person
         # @param external_id [String] the `exernal_id` of the person
         # @param email [String] the `email` of the person
-        # @param strict [Boolean] if should perform a `soft` or a `strict` search. `strict` will avoid repeated email addresses.
+        # @param strict [Boolean] if should perform a `:soft` or a `:strict` search. `strict` will avoid repeated email addresses.
         # @return [Person, nil] the person we were searching, or `nil` if not found.
         def person(id: nil, external_id: nil, email: nil, strict: false)
           init_caches
-          pers = @by_id[id]&.first if id
-          pers = @by_external_id[external_id&.strip]&.first       if !pers && !external_id.to_s.strip.empty?
-
-          # strict prevents taking existing user for searched person with same email
-          # specially useful if the organisation ensures all have external id (no need for email search)
-          if !pers && (!strict || external_id.to_s.strip.empty?)
-            # person still not found and either not strict or no external_id provided
-            pers = @by_users_email[email&.downcase.strip]&.first  if !email.to_s.strip.empty?
-
-            if !pers && !strict && !email.to_s.strip.empty?
-              candidates = @by_non_users_email[email&.downcase.strip] || []
-              raise "Too many non-user candidates (#{candidates.length}) with email '#{email}'" if candidates.length > 1
-              pers = candidates.first
-            end
-
-            pers = @by_external_id[email&.downcase.strip]&.first    if !pers && !email.to_s.strip.empty?
-          end
-
+          # normalize values
+          ext_id   = !external_id.to_s.strip.empty? && external_id.strip
+          email    = !email.to_s.strip.empty? && email.downcase.strip
+
+          pers   = nil
+          pers ||= @by_id[id]&.first
+          pers ||= @by_external_id[ext_id]&.first
+          pers ||= person_by_email(email) unless strict && ext_id
           pers
         end
 
+        # @see Eco::API::Organization::People#person
         def find(object, strict: false)
           id          = attr_value(object, "id")
           external_id = attr_value(object, "external_id")
@@ -176,6 +215,12 @@ module Eco
         end
         # @!endgroup
 
+        # @!group Helper methods
+        def similarity
+          Eco::API::Organization::PeopleSimilarity.new(self.to_a)
+        end
+        # @!endgroup
+
         protected
 
         def on_change
@@ -184,15 +229,46 @@ module Eco
 
         private
 
+        def person_by_email(email, prevent_duplicates: true)
+          return nil unless email
+
+          candidates  = @by_non_users_email[email] || []
+          email_users = @by_users_email[email]     || []
+
+          if pers = email_users.first
+            return pers if candidates.empty?
+            candidates = [pers] + candidates
+          elsif candidates.length == 1
+            return candidates.first
+          end
+
+          if prevent_duplicates && !candidates.empty?
+            msg = "Multiple search results match the criteria."
+            raise MultipleSearchResults.new(msg, candidates: candidates, property: "email")
+          end
+
+          @by_external_id[email]&.first
+        end
+
         def init_caches
           return if @caches_init
           @by_id              = to_h
-          @by_external_id     = to_h('external_id')
-          @by_users_email     = users.to_h('email')
-          @by_non_users_email = non_users.to_h('email')
+          @by_external_id     = no_nil_key(to_h('external_id'))
+          @by_users_email     = no_nil_key(existing_users.to_h('email'))
+          @by_non_users_email = no_nil_key(non_users.to_h('email'))
+          @by_email           = no_nil_key(to_h('email'))
           @caches_init = true
         end
 
+        def existing_users
+          newFrom users.select {|u| !u.account_added?(:original)}
+        end
+
+        def no_nil_key(hash)
+          hash.tap {|h| h.delete(nil)}
+        end
+
+
       end
     end
   end

data/lib/eco/api/organization/people_similarity.rb

@@ -0,0 +1,272 @@
+module Eco
+  module API
+    module Organization
+
+      # Class to find out duplicates in the People Manager
+      #
+      # @attr_writer attribute [String, Proc, nil] the target attribute to be read.
+      class PeopleSimilarity <  Eco::API::Organization::People
+        include Eco::Data::FuzzyMatch
+
+        attr_accessor :attribute
+
+        # @!group Config
+        # @return [String, Proc, nil] the target attribute to be read.
+        def attribute=(attr)
+          @attribute = attr
+        end
+
+        def attribute
+          @attribute ||= :name
+        end
+
+        # Returns the target value to analyse
+        # @param person [Ecoportal::API::V1::Person]
+        def item_value(person)
+          return attr.call(item) if attribute.is_a?(Proc)
+          attr = attribute.to_sym
+          return item.send(attr) if item.respond_to?(attr)
+        end
+
+        # Define the order or relevant of per user matches
+        # @param values[Array<Symbol>] the algorithms' results it should be ordered by
+        #   * Possible values: `:dice`, `:levenshtein`, `:jaro_winkler`, `:ngrams`, `:words_ngrams`, `:chars_position`
+        def order=(values)
+          @order = values
+        end
+
+        def order
+          @order ||= [:words_ngrams, :dice]
+        end
+
+        # Define the order or relevant of per user matches
+        # @param value [Float] the threshold that all of the algorithms should comply with
+        def threshold=(value)
+          @threshold = value
+        end
+
+        def threshold
+          @threshold ||= 0.15
+        end
+
+        # Generates a new object with same config but different base `data`.
+        # @return [Eco::API::Organization::PeopleSimilarity]
+        def newFrom(data)
+          super(data).tap do |simil|
+            simil.threshold = threshold
+            simil.order     = order
+            simil.attribute = attribute
+          end
+        end
+
+        # @!endgroup
+
+        # @!group Searchers
+
+        # It gathers those that have the same `email`
+        # @return [Hash] where `keys` are `email`s and `values` an `Array<Person>`
+        def repeated_emails
+          init_caches
+          @by_email.select do |email, people|
+            people.count > 1
+          end
+        end
+
+        # It returns all people with no name
+        # @return [Eco::API::Organization::PeopleSimilarity]
+        def unnamed
+          select do |person|
+            person.name.to_s.strip.length < 2
+          end.yield_self do |results|
+            newFrom(results)
+          end
+        end
+
+        # It returns all people with no name
+        # @return [Eco::API::Organization::PeopleSimilarity]
+        def named
+          reject do |person|
+            person.name.to_s.strip.length < 2
+          end.yield_self do |results|
+            newFrom(results)
+          end
+        end
+
+        # It returns all the entries with `attribute` empty
+        # @return [Eco::API::Organization::PeopleSimilarity]
+        def blank_attribute
+          select do |person|
+            item_value(person).to_s.strip.length < 2
+          end.yield_self do |results|
+            newFrom(results)
+          end
+        end
+
+        # It returns all the entries with `attribute` **n0t** empty
+        # @return [Eco::API::Organization::PeopleSimilarity]
+        def attribute_present
+          reject do |person|
+            item_value(person).to_s.strip.length < 2
+          end.yield_self do |results|
+            newFrom(results)
+          end
+        end
+
+        # @!endgroup
+
+        # @!group Analisys starters
+
+        # Analyses People bases on `options`
+        # @param needle_read [Proc, Symbol] when the value to read from `needle` object is different to the `:read` (`attribute`).
+        #   This allows to for example, facet `needle.name` (needle_read) against `haystack_item.details[alt_id]` (read).
+        # @param keep_empty [Boolean] to indicate if it should get rid of people with no results (based on threshold)
+        # @return [Hash] where the _keys_ are the people `id`s and the _values_ the `Eco::Data::FuzzyMatch::Results`
+        def analyse(needle_read: nil, keep_empty: false, **options)
+          options = { read: self.attribute }.merge(options)
+          total = count; i = 1
+          each_with_object({}) do |person, results|
+            needle_str = needle_read ? item_string(person, needle_read) : nil
+            results[person.id] = find_all_with_score(person, needle_str: needle_str, **options)
+            print_progress("Analysed", total, i)
+            i += 1
+          end.yield_self do |analysed|
+            analysed = clean_empty(analysed) unless keep_empty
+            #puts "... #{analysed.count} results after cleaning empty"
+            analysed
+          end
+        end
+
+        # @!endgroup
+
+        # @!group Results Treatment
+
+        # Gets a new instance object of this class, with only people in results
+        # @param analysed [Hash] where the _keys_ are the people `id`s and _values_ the `Eco::Data::FuzzyMatch::Results`
+        # @return [Eco::API::Organization::PeopleSimilarity]
+        def newSimilarity(analysed)
+          newFrom(people_in_results(analysed))
+        end
+
+        def people_in_results(analysed)
+          analysed.each_with_object([]) do |(id, results), people|
+            related = results.each_with_object([self[id]]) do |result, related|
+              related << result.match
+            end
+            related.each {|person| people << person unless people.include?(person)}
+          end
+        end
+
+        # Removes from results those that do not have similar entries
+        def clean_empty(analysed)
+          analysed.select do |id, results|
+            !results.empty?
+          end
+        end
+
+        # Helper to do some treatment fo the results
+        # @param analysed [Hash] where the _keys_ are the people `id`s and _values_ the `Eco::Data::FuzzyMatch::Results`
+        # @return [Hash] where the _keys_ are the people `id`s and _values_ the `Eco::Data::FuzzyMatch::Results`
+        def with_analysed(analysed, keep_empty: false)
+          analysed.each_with_object({}) do |(id, results), reanalysed|
+            reanalysed[id] = yield(self[id], results)
+          end.yield_self do |reanalysed|
+            reanalysed = clean_empty(reanalysed) unless keep_empty
+            reanalysed
+          end.tap {|out| "with_analysed... returns #{out.count} records"}
+        end
+
+        # Launches a reanalyis on `analysed` based on `options`
+        # @param analysed [Hash] where the _keys_ are the people `id`s and the _values_ the `Eco::Data::FuzzyMatch::Results`
+        def rearrange(analysed, **options)
+          with_analysed(analysed) do |person, results|
+            results.relevant_results(**options)
+          end
+        end
+
+        # Reanalyses by using a block to treat the needle and item values
+        def reanalyse(analysed, msg: "Reanalysing", **options, &block)
+          options = { read: self.attribute }.merge(options)
+          total = analysed.count; i = 1
+          with_analysed(analysed) do |person, results|
+            print_progress(msg, total, i)
+            i += 1
+            recalculate_results(results, &block)
+          end
+        end
+
+        # Renalyses by ignoring matching words between the `needle` and those found in `results`
+        def ignore_matching_words(analysed, **options)
+          prompt = "Reanalysing by ignoring matching words"
+          reanalyse(analysed, msg: prompt, **options) do |needle_str, item_str, needle, item|
+            self.class.remove_matching_words(needle_str, item_str)
+          end
+        end
+
+        # Renalyses by ignoring matching words between the `needle` and those found in `results`
+        def ignore_matching_words_old(analysed, **options)
+          options = { read: self.attribute }.merge(options)
+          total = analysed.count; i = 1
+          with_analysed(analysed) do |person, results|
+            print_progress("Reanalysing by ignoring matching words", total, i)
+            i += 1
+            ignore_same_words_score(results, **options)
+          end
+        end
+
+        # @!endgroup
+
+        # @!group Reporting Helpers
+
+        # @return [String] well structured text
+        def report(analysed, format: :txt)
+          case
+          when format == :txt
+            analysed.each_with_object("") do |(id, results), out|
+              msg = results.results.map {|r| r.print}.join("\n  ")
+              out << "#{self[id].identify}:\n  " + msg + "\n"
+            end
+          end
+        end
+
+        # @note
+        #   1. Unless `:analysed` is provided, it launches an analysis cutting with Jaro Winker min 0.5
+        #   2. It then re-sorts and cuts based on `options`
+        # @return [Hash] where the _keys_ are the people `id`s and the _values_ the `Eco::Data::FuzzyMatch::Results`
+        def print_analysis(**options)
+          analysed = options[:analysed] || results_with_false_positives.analyse(**options)
+          analysed.each_with_object({}) do |(id, results), out|
+            puts report(analysed)
+          end
+        end
+        # @!endgroup
+
+        protected
+
+        def on_change
+          remove_instance_variable(@fuzzy_match)
+          super
+        end
+
+        private
+
+        def print_progress(msg, total, num)
+          return unless total > 10
+          puts "" unless num > 1
+          @print_msg_len ||= 0
+          percent = (100 * num.to_f / total).round(1)
+          msg     = "  #{msg}: #{percent}% (#{num} of #{total})\r"
+          @print_msg_len = msg.length unless @print_msg_len > msg.length
+          print msg
+          $stdout.flush
+          if percent > 99.9
+            sleep(0.2)
+            print "#{" " * @print_msg_len}\r"
+            $stdout.flush
+          end
+        end
+
+
+      end
+    end
+  end
+end