eco-helpers 2.0.18 → 2.0.24

data/lib/eco/api/common/session/helpers.rb

@@ -0,0 +1,30 @@
+require_relative 'helpers/prompt_user'
+
+module Eco
+  module API
+    module Common
+      module Session
+        module Helpers
+
+          class << self
+            def included(base)
+              base.send(:include, InstanceMethods)
+              base.extend(ClassMethods)
+            end
+          end
+
+
+          module ClassMethods
+
+          end
+
+          module InstanceMethods
+            include Helpers::PromptUser
+            
+          end
+
+        end
+      end
+    end
+  end
+end

data/lib/eco/api/common/session/helpers/prompt_user.rb

@@ -0,0 +1,34 @@
+require 'timeout'
+module Eco
+  module API
+    module Common
+      module Session
+        module Helpers
+          module PromptUser
+
+            def prompt_user(question, default:, explanation: "", timeout: nil)
+              response = if config.run_mode_remote?
+                          default
+                        else
+                          puts explanation
+                          print "#{question} "
+                          if timeout
+                            begin
+                              Timeout::timeout(timeout) { STDIN.gets.chop }
+                            rescue Timeout::Error
+                              default
+                            end
+                          else
+                            STDIN.gets.chop
+                          end
+                        end
+              return response unless block_given?
+              yield(response)
+            end
+
+          end
+        end
+      end
+    end
+  end
+end

data/lib/eco/api/common/version_patches/ecoportal_api/external_person.rb

@@ -18,7 +18,7 @@ module Ecoportal
             entry.to_s(:identify)
           else
             str_id = id ? "id: '#{id}'; " : ""
-            "#{name}' (#{str_id}ext_id: '#{external_id}'; email: '#{email}')"
+            "'#{name}' (#{str_id}ext_id: '#{external_id}'; email: '#{email}')"
           end
         end
 

data/lib/eco/api/common/version_patches/ecoportal_api/internal_person.rb

@@ -11,12 +11,15 @@ module Ecoportal
           original_doc["account"] = JSON.parse(doc["account"])
         end
 
-        def new?
-          !initial_doc["details"] && !initial_doc["account"]
+        def new?(doc = :initial)
+          ref_doc = (doc == :original) ? original_doc : initial_doc
+          !ref_doc["details"] && !ref_doc["account"]
         end
 
-        def account_added?
-          account && !initial_doc["account"]
+        # @return [Boolean] if the account has been added, compared to `doc`
+        def account_added?(doc = :initial)
+          ref_doc = (doc == :original) ? original_doc : initial_doc
+          account && !ref_doc["account"]
         end
 
       end

data/lib/eco/api/common/version_patches/exception.rb

@@ -2,8 +2,11 @@ class ::Exception
   def patch_full_message
     begin
       msg = []
-      msg << "\n#{backtrace.first} \n#{message} (#{self.class.to_s})"
-      backtrace[1..-1].each_with_index {|bt, i| msg << "#{" "*8}#{i+1}: from #{bt}"}
+      tracing   = backtrace ? backtrace : []
+      tracing   = (self.class == SystemStackError) ? tracing[1..30] : tracing[1..-1]
+      tracing ||= []
+      msg << "\n#{tracing.first} \n#{message} (#{self.class.to_s})"
+      tracing.each_with_index {|bt, i| msg << "#{" "*8}#{i+1}: from #{bt}"}
       msg.join("\n")
     rescue Exception => e
       puts "Something is wrong with 'patch_full_message': #{e}"

data/lib/eco/api/microcases/with_each.rb

@@ -12,16 +12,77 @@ module Eco
       # @yieldparam person [Ecoportal::API::V1::Person] the found person that matches `entry`, or a new person otherwise.
       # @return [Eco::API::Organization::People] all the people, including new and existing ones.
       def with_each(entries, people, options)
-        entries.map do |entry|
-          unless person = people.find(entry, strict: micro.strict_search?(options))
-            person = session.new_person
+        @_skip_all_multiple_results = false
+        entries.each_with_object([]) do |entry, scoped|
+          begin
+            unless person = people.find(entry, strict: micro.strict_search?(options))
+              person = session.new_person
+            end
+          rescue Eco::API::Organization::People::MultipleSearchResults => e
+            unless @_skip_all_multiple_results
+              msg    = "\n * When searching this Entry: #{entry.to_s(:identify)}"
+              person = _with_each_prompt_to_select_user(e.append_message(msg), entry: entry)
+            end
+          end
+
+          if person
+            person.entry = entry
+            yield(entry, person) if block_given?
+            scoped << person
           end
-          person.entry = entry
-          yield(entry, person) if block_given?
-          person
         end.yield_self {|all_people| people.newFrom all_people.uniq}
       end
 
+      private
+
+      def _with_each_prompt_to_select_user(error, entry: nil, increase_count: true)
+        unless error.is_a?(Eco::API::Organization::People::MultipleSearchResults)
+          raise "Expecting Eco::API::Organization::People::MultipleSearchResults. Given: #{error.class}"
+        end
+        @_with_each_prompts  = 0 unless instance_variable_defined?(:@_with_each_prompts)
+        @_with_each_prompts += 1 if increase_count
+
+        lines = []
+        lines << "\n(#{@_with_each_prompts}) " + error.to_s + "\n"
+        lines << "  #index   - Select the correct person by its number index among the list above."
+        lines << "  (I)      - Just Skip/Ignore this one. I will deal with that input entry in another launch."
+        lines << "  (A)      - Ignore all the rest of input entries with this problem."
+        lines << "  (C)      - Create a new person."
+        lines << "  (B)      - Just break this script. I need to change the input file :/"
+
+        prompt_user("Type one option (#number/I/A/C/B):", explanation: lines.join("\n"), default: "I") do |res|
+          res = res.upcase
+          case
+          when res.start_with?("I")
+            logger.info "Ignoring entry... #{entry.to_s(:identify) if entry}"
+            nil
+          when res.start_with?("A")
+            logger.info "All input entries with this same issue will be ignored for this launch"
+            @_skip_all_multiple_results = true
+            nil
+          when res.start_with?("C")
+            logger.info "Creating new person...#{"for entry #{entry.to_s(:identify)}" if entry}"
+            session.new_person
+          when res.start_with?("B")
+            raise error
+          when res && !res.empty? && (pos = res.to_i rescue nil) && (pos < error.candidates.length)
+            error.candidate(pos).tap do |person|
+              logger.info "Thanks!! You selected #{person.identify}"
+              sleep(1.5)
+            end
+          else
+            if pos.is_a?(Numeric) && (pos >= error.candidates.length)
+              print "#{pos} is not a number in the range. "
+            else
+              print "#{res} is not an option. "
+            end
+            puts "Please select one of the offered options..."
+            sleep(1)
+            _with_each_prompt_to_select_user(error, increase_count: false, entry: entry)
+          end
+        end
+      end
+
     end
   end
 end

data/lib/eco/api/microcases/with_each_present.rb

@@ -15,8 +15,10 @@ module Eco
       def with_each_present(entries, people, options, log_starter: false)
         found = []
         micro.with_each(entries, people, options) do |entry, person|
-          if person.new? && log_starter
-            session.logger.error("This person does not exist: #{entry.to_s(:identify)}")
+          if person.new?
+            if log_starter
+              session.logger.error("This person does not exist: #{entry.to_s(:identify)}")
+            end
             next
           end
           found << person

data/lib/eco/api/microcases/with_each_starter.rb

@@ -15,8 +15,10 @@ module Eco
       def with_each_starter(entries, people, options, log_present: false)
         starters = []
         micro.with_each(entries, people, options) do |entry, person|
-          if !person.new? && log_present
-            session.logger.error("This person (id: '#{person.id}') already exists: #{entry.to_s(:identify)}")
+          if !person.new?
+            if log_present
+              session.logger.error("This person (id: '#{person.id}') already exists: #{entry.to_s(:identify)}")
+            end
             next
           end
           starters << person

data/lib/eco/api/organization.rb

@@ -9,7 +9,7 @@ require_relative 'organization/tag_tree'
 require_relative 'organization/presets_factory'
 require_relative 'organization/preferences'
 require_relative 'organization/people'
-require_relative 'organization/people_analytics'
+require_relative 'organization/people_similarity'
 require_relative 'organization/person_schemas'
 require_relative 'organization/policy_groups'
 require_relative 'organization/login_providers'

data/lib/eco/api/organization/people.rb

@@ -2,6 +2,43 @@ module Eco
   module API
     module Organization
       class People <  Eco::Language::Models::Collection
+        # Error class that allows to handle cases where multiple people were found for the same criterion.
+        # @note its main purpose to prevent the creation of duplicates or override information between different people.
+        class MultipleSearchResults < StandardError
+          attr_reader :candidates, :property
+          # @param msg [String] the basic message error.
+          # @param candiates [Array<Person>] the people that match the same search criterion.
+          # @param property [String] the property of the person model that triggered the error (base of the search criterion).
+          def initialize(msg, candidates: [], property: "email")
+            @candidates = candidates
+            @property   = property
+            super(msg + " " + candidates_summary)
+          end
+
+          # @param with_index [Boolean] to add an index to each candidate description.
+          # @return [Array<String>] the `candidates` identified
+          def identify_candidates(with_index: false)
+            candidates.map.each_with_index do |person, i|
+              index = with_index     ? "#{i}. " : ""
+              msg   = person.account ? (person.account_added? ? "(new user)" : "(user)") :  "(no account)"
+              "#{index}#{msg} #{person.identify}"
+            end
+          end
+
+          # @return [Person] the `candidate` in the `index` position
+          def candidate(index)
+            candidates[index]
+          end
+
+          private
+
+          def candidates_summary
+            lines = ["The following people have the same '#{property}':"]
+            lines.concat(identify_candidates(with_index: true)).join("\n  ")
+          end
+
+        end
+
         # build the shortcuts of Collection
         attr_presence :account, :details
         attr_collection :id, :external_id, :email, :name, :supervisor_id
@@ -78,34 +115,36 @@ module Eco
         # @!group Searchers
 
         # It searches a person using the parameters given.
+        # @note This is how the search function actually works:
+        #   1. if eP `id` is given, returns the person (if found), otherwise...
+        #   2. if `external_id` is given, returns the person (if found), otherwise...
+        #   3. if `strict` is `false` and `email` is given:
+        #     - if there is only 1 person with that email, returns that person, otherwise...
+        #     - if found but, there are many candidates, it raises MultipleSearchResults error
+        #     - if person `external_id` matches `email`, returns that person
+        # @raise MultipleSearchResults if there are multiple people with the same `email`
+        #   and there's no other criteria to find the person. It only gets to this point if
+        #   `external_id` was **not** provided and we are **not** in 'strict' search mode.
+        #   However, it could be we were in `strict` mode and `external_id` was not provided.
         # @param id [String] the `internal id` of the person
         # @param external_id [String] the `exernal_id` of the person
         # @param email [String] the `email` of the person
-        # @param strict [Boolean] if should perform a `soft` or a `strict` search. `strict` will avoid repeated email addresses.
+        # @param strict [Boolean] if should perform a `:soft` or a `:strict` search. `strict` will avoid repeated email addresses.
         # @return [Person, nil] the person we were searching, or `nil` if not found.
         def person(id: nil, external_id: nil, email: nil, strict: false)
           init_caches
-          pers = @by_id[id]&.first if id
-          pers = @by_external_id[external_id&.strip]&.first       if !pers && !external_id.to_s.strip.empty?
-
-          # strict prevents taking existing user for searched person with same email
-          # specially useful if the organisation ensures all have external id (no need for email search)
-          if !pers && (!strict || external_id.to_s.strip.empty?)
-            # person still not found and either not strict or no external_id provided
-            pers = @by_users_email[email&.downcase.strip]&.first  if !email.to_s.strip.empty?
-
-            if !pers && !strict && !email.to_s.strip.empty?
-              candidates = @by_non_users_email[email&.downcase.strip] || []
-              raise "Too many non-user candidates (#{candidates.length}) with email '#{email}'" if candidates.length > 1
-              pers = candidates.first
-            end
-
-            pers = @by_external_id[email&.downcase.strip]&.first    if !pers && !email.to_s.strip.empty?
-          end
-
+          # normalize values
+          ext_id   = !external_id.to_s.strip.empty? && external_id.strip
+          email    = !email.to_s.strip.empty? && email.downcase.strip
+
+          pers   = nil
+          pers ||= @by_id[id]&.first
+          pers ||= @by_external_id[ext_id]&.first
+          pers ||= person_by_email(email) unless strict && ext_id
           pers
         end
 
+        # @see Eco::API::Organization::People#person
         def find(object, strict: false)
           id          = attr_value(object, "id")
           external_id = attr_value(object, "external_id")
@@ -177,8 +216,8 @@ module Eco
         # @!endgroup
 
         # @!group Helper methods
-        def analytics
-          Eco::API::Organization::PeopleAnalytics.new(self.to_a)
+        def similarity
+          Eco::API::Organization::PeopleSimilarity.new(self.to_a)
         end
         # @!endgroup
 
@@ -190,16 +229,46 @@ module Eco
 
         private
 
+        def person_by_email(email, prevent_duplicates: true)
+          return nil unless email
+
+          candidates  = @by_non_users_email[email] || []
+          email_users = @by_users_email[email]     || []
+
+          if pers = email_users.first
+            return pers if candidates.empty?
+            candidates = [pers] + candidates
+          elsif candidates.length == 1
+            return candidates.first
+          end
+
+          if prevent_duplicates && !candidates.empty?
+            msg = "Multiple search results match the criteria."
+            raise MultipleSearchResults.new(msg, candidates: candidates, property: "email")
+          end
+
+          @by_external_id[email]&.first
+        end
+
         def init_caches
           return if @caches_init
           @by_id              = to_h
-          @by_external_id     = to_h('external_id')
-          @by_users_email     = users.to_h('email')
-          @by_non_users_email = non_users.to_h('email')
-          @by_email           = to_h('email')
+          @by_external_id     = no_nil_key(to_h('external_id'))
+          @by_users_email     = no_nil_key(existing_users.to_h('email'))
+          @by_non_users_email = no_nil_key(non_users.to_h('email'))
+          @by_email           = no_nil_key(to_h('email'))
           @caches_init = true
         end
 
+        def existing_users
+          newFrom users.select {|u| !u.account_added?(:original)}
+        end
+
+        def no_nil_key(hash)
+          hash.tap {|h| h.delete(nil)}
+        end
+
+
       end
     end
   end

data/lib/eco/api/organization/people_similarity.rb

@@ -0,0 +1,272 @@
+module Eco
+  module API
+    module Organization
+
+      # Class to find out duplicates in the People Manager
+      #
+      # @attr_writer attribute [String, Proc, nil] the target attribute to be read.
+      class PeopleSimilarity <  Eco::API::Organization::People
+        include Eco::Data::FuzzyMatch
+
+        attr_accessor :attribute
+
+        # @!group Config
+        # @return [String, Proc, nil] the target attribute to be read.
+        def attribute=(attr)
+          @attribute = attr
+        end
+
+        def attribute
+          @attribute ||= :name
+        end
+
+        # Returns the target value to analyse
+        # @param person [Ecoportal::API::V1::Person]
+        def item_value(person)
+          return attr.call(item) if attribute.is_a?(Proc)
+          attr = attribute.to_sym
+          return item.send(attr) if item.respond_to?(attr)
+        end
+
+        # Define the order or relevant of per user matches
+        # @param values[Array<Symbol>] the algorithms' results it should be ordered by
+        #   * Possible values: `:dice`, `:levenshtein`, `:jaro_winkler`, `:ngrams`, `:words_ngrams`, `:chars_position`
+        def order=(values)
+          @order = values
+        end
+
+        def order
+          @order ||= [:words_ngrams, :dice]
+        end
+
+        # Define the order or relevant of per user matches
+        # @param value [Float] the threshold that all of the algorithms should comply with
+        def threshold=(value)
+          @threshold = value
+        end
+
+        def threshold
+          @threshold ||= 0.15
+        end
+
+        # Generates a new object with same config but different base `data`.
+        # @return [Eco::API::Organization::PeopleSimilarity]
+        def newFrom(data)
+          super(data).tap do |simil|
+            simil.threshold = threshold
+            simil.order     = order
+            simil.attribute = attribute
+          end
+        end
+
+        # @!endgroup
+
+        # @!group Searchers
+
+        # It gathers those that have the same `email`
+        # @return [Hash] where `keys` are `email`s and `values` an `Array<Person>`
+        def repeated_emails
+          init_caches
+          @by_email.select do |email, people|
+            people.count > 1
+          end
+        end
+
+        # It returns all people with no name
+        # @return [Eco::API::Organization::PeopleSimilarity]
+        def unnamed
+          select do |person|
+            person.name.to_s.strip.length < 2
+          end.yield_self do |results|
+            newFrom(results)
+          end
+        end
+
+        # It returns all people with no name
+        # @return [Eco::API::Organization::PeopleSimilarity]
+        def named
+          reject do |person|
+            person.name.to_s.strip.length < 2
+          end.yield_self do |results|
+            newFrom(results)
+          end
+        end
+
+        # It returns all the entries with `attribute` empty
+        # @return [Eco::API::Organization::PeopleSimilarity]
+        def blank_attribute
+          select do |person|
+            item_value(person).to_s.strip.length < 2
+          end.yield_self do |results|
+            newFrom(results)
+          end
+        end
+
+        # It returns all the entries with `attribute` **n0t** empty
+        # @return [Eco::API::Organization::PeopleSimilarity]
+        def attribute_present
+          reject do |person|
+            item_value(person).to_s.strip.length < 2
+          end.yield_self do |results|
+            newFrom(results)
+          end
+        end
+
+        # @!endgroup
+
+        # @!group Analisys starters
+
+        # Analyses People bases on `options`
+        # @param needle_read [Proc, Symbol] when the value to read from `needle` object is different to the `:read` (`attribute`).
+        #   This allows to for example, facet `needle.name` (needle_read) against `haystack_item.details[alt_id]` (read).
+        # @param keep_empty [Boolean] to indicate if it should get rid of people with no results (based on threshold)
+        # @return [Hash] where the _keys_ are the people `id`s and the _values_ the `Eco::Data::FuzzyMatch::Results`
+        def analyse(needle_read: nil, keep_empty: false, **options)
+          options = { read: self.attribute }.merge(options)
+          total = count; i = 1
+          each_with_object({}) do |person, results|
+            needle_str = needle_read ? item_string(person, needle_read) : nil
+            results[person.id] = find_all_with_score(person, needle_str: needle_str, **options)
+            print_progress("Analysed", total, i)
+            i += 1
+          end.yield_self do |analysed|
+            analysed = clean_empty(analysed) unless keep_empty
+            #puts "... #{analysed.count} results after cleaning empty"
+            analysed
+          end
+        end
+
+        # @!endgroup
+
+        # @!group Results Treatment
+
+        # Gets a new instance object of this class, with only people in results
+        # @param analysed [Hash] where the _keys_ are the people `id`s and _values_ the `Eco::Data::FuzzyMatch::Results`
+        # @return [Eco::API::Organization::PeopleSimilarity]
+        def newSimilarity(analysed)
+          newFrom(people_in_results(analysed))
+        end
+
+        def people_in_results(analysed)
+          analysed.each_with_object([]) do |(id, results), people|
+            related = results.each_with_object([self[id]]) do |result, related|
+              related << result.match
+            end
+            related.each {|person| people << person unless people.include?(person)}
+          end
+        end
+
+        # Removes from results those that do not have similar entries
+        def clean_empty(analysed)
+          analysed.select do |id, results|
+            !results.empty?
+          end
+        end
+
+        # Helper to do some treatment fo the results
+        # @param analysed [Hash] where the _keys_ are the people `id`s and _values_ the `Eco::Data::FuzzyMatch::Results`
+        # @return [Hash] where the _keys_ are the people `id`s and _values_ the `Eco::Data::FuzzyMatch::Results`
+        def with_analysed(analysed, keep_empty: false)
+          analysed.each_with_object({}) do |(id, results), reanalysed|
+            reanalysed[id] = yield(self[id], results)
+          end.yield_self do |reanalysed|
+            reanalysed = clean_empty(reanalysed) unless keep_empty
+            reanalysed
+          end.tap {|out| "with_analysed... returns #{out.count} records"}
+        end
+
+        # Launches a reanalyis on `analysed` based on `options`
+        # @param analysed [Hash] where the _keys_ are the people `id`s and the _values_ the `Eco::Data::FuzzyMatch::Results`
+        def rearrange(analysed, **options)
+          with_analysed(analysed) do |person, results|
+            results.relevant_results(**options)
+          end
+        end
+
+        # Reanalyses by using a block to treat the needle and item values
+        def reanalyse(analysed, msg: "Reanalysing", **options, &block)
+          options = { read: self.attribute }.merge(options)
+          total = analysed.count; i = 1
+          with_analysed(analysed) do |person, results|
+            print_progress(msg, total, i)
+            i += 1
+            recalculate_results(results, &block)
+          end
+        end
+
+        # Renalyses by ignoring matching words between the `needle` and those found in `results`
+        def ignore_matching_words(analysed, **options)
+          prompt = "Reanalysing by ignoring matching words"
+          reanalyse(analysed, msg: prompt, **options) do |needle_str, item_str, needle, item|
+            self.class.remove_matching_words(needle_str, item_str)
+          end
+        end
+
+        # Renalyses by ignoring matching words between the `needle` and those found in `results`
+        def ignore_matching_words_old(analysed, **options)
+          options = { read: self.attribute }.merge(options)
+          total = analysed.count; i = 1
+          with_analysed(analysed) do |person, results|
+            print_progress("Reanalysing by ignoring matching words", total, i)
+            i += 1
+            ignore_same_words_score(results, **options)
+          end
+        end
+
+        # @!endgroup
+
+        # @!group Reporting Helpers
+
+        # @return [String] well structured text
+        def report(analysed, format: :txt)
+          case
+          when format == :txt
+            analysed.each_with_object("") do |(id, results), out|
+              msg = results.results.map {|r| r.print}.join("\n  ")
+              out << "#{self[id].identify}:\n  " + msg + "\n"
+            end
+          end
+        end
+
+        # @note
+        #   1. Unless `:analysed` is provided, it launches an analysis cutting with Jaro Winker min 0.5
+        #   2. It then re-sorts and cuts based on `options`
+        # @return [Hash] where the _keys_ are the people `id`s and the _values_ the `Eco::Data::FuzzyMatch::Results`
+        def print_analysis(**options)
+          analysed = options[:analysed] || results_with_false_positives.analyse(**options)
+          analysed.each_with_object({}) do |(id, results), out|
+            puts report(analysed)
+          end
+        end
+        # @!endgroup
+
+        protected
+
+        def on_change
+          remove_instance_variable(@fuzzy_match)
+          super
+        end
+
+        private
+
+        def print_progress(msg, total, num)
+          return unless total > 10
+          puts "" unless num > 1
+          @print_msg_len ||= 0
+          percent = (100 * num.to_f / total).round(1)
+          msg     = "  #{msg}: #{percent}% (#{num} of #{total})\r"
+          @print_msg_len = msg.length unless @print_msg_len > msg.length
+          print msg
+          $stdout.flush
+          if percent > 99.9
+            sleep(0.2)
+            print "#{" " * @print_msg_len}\r"
+            $stdout.flush
+          end
+        end
+
+
+      end
+    end
+  end
+end