eco-helpers 2.0.18 → 2.0.24

data/lib/eco/api/organization/person_schemas.rb

@@ -28,7 +28,11 @@ module Eco
         end
 
         def schema(id_name)
-          @by_id.fetch(schema_id(id_name), nil)
+          self[id_name]
+        end
+
+        def [](id_name)
+          @by_id[schema_id(id_name)]
         end
 
         private

data/lib/eco/api/organization/policy_groups.rb

@@ -44,7 +44,11 @@ module Eco
         end
 
         def policy_group(id_name)
-          @by_id.fetch(policy_group_id(id_name), nil)
+          self[id_name]
+        end
+
+        def [](id_name)
+          @by_id[policy_group_id(id_name)]
         end
 
         def user_pg_ids(initial: [], final: [], non_custom: (non_custom_not_used = true; []), preserve_custom: true)

data/lib/eco/api/organization/tag_tree.rb

@@ -42,6 +42,39 @@ module Eco
           init_hashes
         end
 
+        # Updates the tag of the current tree
+        def tag=(value)
+          @tag = value
+        end
+
+        # @return [Eco::API::Organization::TagTree]
+        def dup
+          self.class.new(as_json)
+        end
+
+        # @return [Array] with the differences
+        def diff(tagtree, differences: {}, level: 0, **options)
+          require 'hashdiff'
+          Hashdiff.diff(self.as_json, tagtree.as_json, **options.slice(:array_path, :similarity, :use_lcs))
+        end
+
+        def top?
+          depth == -1
+        end
+
+        # @return [Array[Hash]] where `Hash` is a `node` `{"tag" => TAG, "nodes": Array[Hash]}`
+        def as_json
+          nodes_json = nodes.map {|node| node.as_json}
+          if top?
+            nodes_json
+          else
+            {
+              "tag"   => tag,
+              "nodes" => nodes_json
+            }
+          end
+        end
+
         # @return [Boolean] `true` if there are tags in the node, `false` otherwise.
         def empty?
           @has_tags.empty?

data/lib/eco/api/session.rb

@@ -66,6 +66,16 @@ module Eco
         @presets_factory ||= Eco::API::Organization::PresetsFactory.new(enviro: enviro)
       end
 
+      # @return [Eco::Data::Mapper] the mappings between the internal and external attribute/property names.
+      def fields_mapper
+        return @fields_mapper if instance_variable_defined?(:@fields_mapper)
+        mappings = []
+        if map_file = config.people.fields_mapper
+          mappings = map_file ? file_manager.load_json(map_file) : []
+        end
+        @fields_mapper = Eco::Data::Mapper.new(mappings)
+      end
+
       # Helper to obtain a EntryFactory
       # @param schema [String, Ecoportal::API::V1::PersonSchema] `schema` to which associate the EntryFactory,
       #  where `String` can be the _name_ or the _id_ of the schema.
@@ -74,17 +84,16 @@ module Eco
       def entry_factory(schema: nil)
         schema = to_schema(schema) || self.schema
         return @entry_factories[schema&.id] if @entry_factories.key?(schema&.id)
-
-        mappings = []
-        if map_file = config.people.fields_mapper
-          mappings = map_file ? file_manager.load_json(map_file) : []
+        unless @entry_factories.empty?
+          @entry_factories[schema&.id] = @entry_factories.values.first.newFactory(schema: schema)
+          return @entry_factories[schema&.id]
         end
 
         @entry_factories[schema&.id] = Eco::API::Common::People::EntryFactory.new(
           enviro,
           schema:        schema,
           person_parser: config.people.parser,
-          attr_map:      Eco::Data::Mapper.new(mappings)
+          attr_map:      fields_mapper
         )
       end
 
@@ -103,11 +112,13 @@ module Eco
       # @param attr [String] type (`Symbol`) or attribute (`String`) to target a specific parser.
       # @param source [Any] source value to be parsed.
       # @param phase [Symbol] the phase when this parser should be active.
-      def parse_attribute(attr, source, phase = :internal)
+      # @param phase [Symbol] the phase when this parser should be active.
+      # @return [Object] the parsed attribute.
+      def parse_attribute(attr, source, phase = :internal, deps: {})
         unless parsers = entry_factory.person_parser
           raise "There are no parsers defined"
         end
-        parsers.parse(attr, source, phase)
+        parsers.parse(attr, source, phase, deps: deps)
       end
 
       # @see Eco::API::Common::People::EntryFactory#export
@@ -127,7 +138,7 @@ module Eco
       # @see Eco::API::Common::People::EntryFactory#new
       # @return [Eco::API::Common::People::PersonEntry] parsed entry.
       def new_entry(data, dependencies: {})
-        entry_factory.new(data, dependencies: dependencies)
+        entry_factory(schema: data&.details&.schema_id).new(data, dependencies: dependencies)
       end
 
       # @see Eco::API::Common::People::EntryFactory#entries

data/lib/eco/api/session/batch.rb

@@ -136,11 +136,13 @@ module Eco
             block.call
           rescue error_type => e
             raise unless retries_left > 0
-            print "Batch TimeOut. You have #{retries_left} retries left. Do you want to retry (y/N)? "
-            if (res = STDIN.gets.chomp) && res[0].downcase == "y"
-              offer_retry_on(error_type, retries_left - 1, &block)
-            else
-              raise
+            explanation = "Batch TimeOut. You have #{retries_left} retries left."
+            prompt_user("Do you want to retry (y/N)?", explanation, default: "Y", timeout: 10) do |response|
+              if response.upcase.start_with?("Y")
+                offer_retry_on(error_type, retries_left - 1, &block)
+              else
+                raise
+              end
             end
           end
         end

data/lib/eco/api/session/batch/job.rb

@@ -164,12 +164,17 @@ module Eco
           # @return [Eco::API::Session::Batch::Status]
           def launch(simulate: false)
             pqueue    = processed_queue
-            @requests = pqueue.map {|e| as_update(e)}
+            @requests = as_update(pqueue)
             pre_checks(requests, simulate: simulate)
 
-            unless simulate
+            if simulate
+              if options.dig(:requests, :backup)
+                req_backup = as_update(pqueue, add_feedback: false)
+                backup_update(req_backup, simulate: simulate)
+              end
+            else
               if pqueue.length > 0
-                req_backup = pqueue.map {|e| as_update(e, add_feedback: false)}
+                req_backup = as_update(pqueue, add_feedback: false)
                 backup_update(req_backup)
                 session.batch.launch(pqueue, method: type).tap do |job_status|
                   @status     = job_status
@@ -220,13 +225,26 @@ module Eco
             end.join("\n")
           end
 
-          def as_update(*args)
-            feedback.as_update(*args)
+          def as_update(data, *args)
+            if data.is_a?(Array)
+              data.map do |e|
+                feedback.as_update(e, *args)
+              end.compact.select {|e| e && !e.empty?}
+            else
+              feedback.as_update(data, *args)
+            end
           end
 
           def processed_queue
             @queue.each {|e| @callbacks[e].call(e) if @callbacks.key?(e) }
-            apply_policies(api_included(@queue)).select {|e| !as_update(e).empty?}
+            apply_policies(api_included(@queue)).select do |e|
+              !as_update(e).empty?
+            end.select do |e|
+              next true unless e.is_a?(Ecoportal::API::V1::Person)
+              next true unless e.new?
+              # new people should either have account or details
+              e.account || e.details
+            end
           end
 
           # if there is a config definition to exclude entries
@@ -235,7 +253,13 @@ module Eco
           def api_included(full_queue)
             return full_queue if type == :create
             return full_queue unless excluded = session.config.people.api_excluded
-            full_queue.select {|entry| !excluded.call(entry, session, options, self)}
+            if options.dig(:include, :only_excluded)
+              full_queue.select {|entry| excluded.call(entry, session, options, self)}
+            elsif options.dig(:include, :excluded)
+              full_queue
+            else
+              full_queue.select {|entry| !excluded.call(entry, session, options, self)}
+            end
           end
 
           # Applies the changes introduced by api policies
@@ -307,9 +331,10 @@ module Eco
           end
 
           # Keep a copy of the requests for future reference
-          def backup_update(requests)
+          def backup_update(requests, simulate: false)
+            dry_run   = simulate ? "_dry_run" : ""
             dir       = config.people.requests_folder
-            file      = File.join(dir, "#{type}_data.json")
+            file      = File.join(dir, "#{type}_data#{dry_run}.json")
             file_manager.save_json(requests, file, :timestamp)
           end
 

data/lib/eco/api/usecases.rb

@@ -2,7 +2,7 @@ module Eco
   module API
     class UseCases
 
-      class UnkownCase < Exception
+      class UnkownCase < StandardError
         def initialize(msg = nil, case_name: nil, type: nil)
           msg ||= "Unkown case"
           msg += ". Case name '#{case_name}'" if case_name
@@ -11,7 +11,7 @@ module Eco
         end
       end
 
-      class AmbiguousCaseReference < Exception
+      class AmbiguousCaseReference < StandardError
         def initialize(msg = nil, case_name: nil)
           msg ||= "You must specify type when there are multiple cases with same name"
           msg += ". Case name '#{case_name}'" if case_name

data/lib/eco/api/usecases/base_case.rb

@@ -4,7 +4,7 @@ module Eco
       # Core class of UseCases. It basically defines and manages allowed `types`
       class BaseCase
 
-        class InvalidType < Exception
+        class InvalidType < StandardError
           def initialize(msg = nil, type:, types:)
             msg ||= "Invalid type."
             msg = "Given type '#{type}'. Valid types: #{types}"
@@ -13,7 +13,7 @@ module Eco
         end
 
         extend Eco::API::Common::ClassHelpers
-        
+
         @types = [:import, :filter, :transform, :sync, :error_handler, :export, :other]
 
         class << self

data/lib/eco/api/usecases/base_io.rb

@@ -5,6 +5,19 @@ module Eco
       class BaseIO < BaseCase
         @types  = BaseCase.types
 
+        class MissingParameter < StandardError
+          attr_reader :type, :required, :given
+
+          def initialize(msg = nil, type: nil, required:, given:)
+            @type     = type
+            @required = required
+            @given    = given
+            msg += " of type '#{type}'" if type
+            msg += " requires an object '#{required}'. Given: #{given}."
+            super(msg)
+          end
+        end
+
         class << self
           def input_required?(type)
             !valid_type?(type) || [:import, :sync].include?(type)
@@ -80,13 +93,13 @@ module Eco
         def validate_args(input:, people:, session:, options:)
           case
           when !session.is_a?(Eco::API::Session)
-            raise "A UseCase needs a Session object. Given: #{session}"
+            raise MissingParameter.new("UseCase", required: :session, given: session.class)
           when input_required? && !input
-            raise "UseCase of type '#{type}' requires a valid input. None given"
+            raise MissingParameter.new("UseCase", type: type, required: :input, given: input.class)
           when people_required? && !people.is_a?(Eco::API::Organization::People)
-            raise "UseCase of type '#{type}' requires a People object. Given: #{people}"
+            raise MissingParameter.new("UseCase", type: type, required: :people, given: people.class)
           when !options || (options && !options.is_a?(Hash))
-            raise "To inject dependencies via ':options' it should be a Hash object. Given: #{options}"
+            raise MissingParameter.new("Use Case options", required: :Hash, given: options.class)
           end
           true
         end

data/lib/eco/api/usecases/default_cases.rb

@@ -13,6 +13,7 @@ require_relative 'default_cases/abstract_policygroup_abilities_case.rb'
 require_relative 'default_cases/analyse_people_case'
 require_relative 'default_cases/append_usergroups_case'
 require_relative 'default_cases/change_email_case'
+require_relative 'default_cases/clean_unknown_tags_case'
 require_relative 'default_cases/codes_to_tags_case'
 require_relative 'default_cases/create_case'
 require_relative 'default_cases/create_details_case'

data/lib/eco/api/usecases/default_cases/analyse_people_case.rb

@@ -5,71 +5,218 @@ class Eco::API::UseCases::DefaultCases::AnalysePeople < Eco::API::Common::Loader
   attr_reader :session, :people, :options
 
   def main(people, session, options, usecase)
+    options[:end_get] = false
     @session = session; @options = options; @people = people
 
-    save!(cyclic_sets)
+    case
+    when case_options[:identify_duplicates]
+      identify_duplicates
+    when case_options[:identify_unnamed]
+      identify_unnamed
+    else
+      session.logger.info("No analysis operation was specified")
+    end.tap do |people_involved|
+      if people_involved
+        to_csv(people_involved) if to_csv?
+        create_people_backup(people_involved) if results_people_backup?
+      end
+    end
   end
 
   private
 
-  def identify_double_ups
-    analytics.similarity
-    
+  def identify_unnamed
+    similarity_analytics.unnamed.tap do |unnamed|
+      if unnamed.empty?
+        session.logger.info("There were no people with no name!!")
+      end
+    end
   end
 
-  def analytics
-    @analytics ||= people.analytics
+  def identify_duplicates
+    analysed = similarity_screening
+    if case_options[:ignore_matching_words]
+      puts "Fine tune results by ignoring matching words..."
+      analysed = strict_similarity(analysed)
+    end
+
+    similarity_analytics.newSimilarity(analysed).tap do |related_people|
+      if related_people.empty?
+        session.logger.info("There were no possible duplicates identified!!")
+      else
+        report = similarity_analytics.report(analysed, format: :txt)
+        save!(report)
+      end
+    end
   end
 
-  def file
-    @file ||= options.dig(:output, :file) || "analytics.txt"
+  def strict_similarity(analysed)
+    similarity_analytics.ignore_matching_words(analysed, **{
+      threshold: 0.5,
+      order: [:ngrams]
+    })
   end
 
-  def save!(data)
-    if data.empty?
-      session.logger.info("There were no cyclic supervisors identified!!")
-      return
+  def similarity_screening
+    similarity_analytics.attribute = field_similarity
+    options = {
+      threshold: 0.4,
+      order:     [:average, :dice]
+    }.tap do |opts|
+      opts.merge!(needle_read: facet_field_proc) if facet_field?
+      opts.merge!(unique_words: true) if unique_words?
+    end
+    analysed = similarity_analytics.analyse(**options)
+    puts "Got #{analysed.count} results after basic screening with #{options}"
+
+    return analysed if case_options[:only_screening]
+    options = {threshold: 0.5, order: [:average]}
+    puts "Going to rearrange results... with #{options}"
+    similarity_analytics.rearrange(analysed, **options).tap do |analysed|
+      puts "... got #{analysed.count} results after rearranging"
     end
+  end
+
+  def similarity_analytics
+    @analytics ||= people.similarity
+  end
+
+  def create_people_backup(cut = people, file = results_people_backup)
+    session.file_manager.save_json(cut, file)
+  end
+
+  def to_csv(data = people, file = csv_file)
+    opts = {}
+    opts.deep_merge!(export: {file: {name: file, format: :csv}})
+    opts.deep_merge!(export: {options: {nice_header: true}})
+    opts.deep_merge!(export: {options: {internal_names: true}})
+    #opts.deep_merge!(export: {options: {split_schemas: true}})
+    session.process_case("to-csv", type: :export, people: data, options: opts.merge(options.slice(:export)))
+  end
+
+  def unique_words?
+    case_options[:unique_words]
+  end
+
+  def field_similarity
+    return :name unless use_field?
+    use_field_proc
+  end
+
+  def use_field_proc
+    proc_value_access(use_field)
+  end
+
+  def facet_field_proc
+    proc_value_access(facet_field)
+  end
+
+  def use_field
+    case_options.dig(:use_field)
+  end
 
-    ext  = File.extname(file).downcase.delete(".")
+  def use_field?
+    !!use_field
+  end
 
-    File.open(file, "w") do |fd|
+  def facet_field
+    case_options.dig(:facet_field)
+  end
+
+  def facet_field?
+    !!facet_field
+  end
+
+  def csv_file
+    case_options.dig(:csv_file)
+  end
+
+  def to_csv?
+    !!csv_file
+  end
+
+  def results_people_backup
+    case_options.dig(:backup_people)
+  end
+
+  def results_people_backup?
+    !!results_people_backup
+  end
+
+  def case_options
+    options.dig(:usecase, :analyse_people) || {}
+  end
+
+  def output_file
+    @output_file ||= options.dig(:output, :file) || "analytics.txt"
+  end
+
+  def save!(data)
+    ext  = File.extname(output_file).downcase.delete(".")
+    session.logger.info("Generating file '#{output_file}'")
+    File.open(output_file, "w") do |fd|
       if ext == "txt"
-        create_file(data, file: file, format: :txt)
+        fd << data
       elsif ext == "html"
         puts "html is still not supported"
         exit(1)
-        create_file(data, file: file, format: :html)
       elsif ext == "json"
         puts "json is still not supported"
         exit(1)
-        create_file(data, file: file, format: :json)
       end
     end
   end
 
-  def create_file(sets, file:, format: :txt)
-    File.open(file, "w") do |fd|
-      fd << sets_to_str(sets, format: format)
+  # A way to use command line to specify part
+  # => i.e. details[first-name] AND details[surname]
+  def proc_value_access(expression)
+    #return expression.to_sym if expression.start_with?(":")
+    subexpressions = expression.split(" AND ")
+    Proc.new do |person|
+      values = subexpressions.map {|exp| attribute_access(person, exp)}
+      values.compact.join(" ")
     end
-    puts "Generated file #{file}"
   end
 
-  def sets_to_str(sets, format: :txt)
-    raise "Required Array. Given: #{sets.class}" unless sets.is_a?(Array)
-    "".tap do |str|
-      sets.each do |set|
-        str << set_to_str(set, format: format)
+  # A way to use command line to specify part
+  # => i.e. person.details[first-name]
+  def attribute_access(person, expression)
+    parts = expression.split(".")
+    parts_to_value(person, parts).tap do |value|
+      unless value.is_a?(String) || !value
+        raise "Something is wrong with #{expression} to parts #{parts}. Expecting String, obtained: #{value.class}"
       end
     end
   end
 
-  def set_to_str(set, lev: 0, format: :txt)
-    raise "Required Array. Given: #{set.class}" unless set.is_a?(Array)
-    "".tap do |str|
-      entry = set.shift
-      str << "#{"  " * lev}#{(lev > 0)? "+-#{lev}- " : ""}#{entry.name} (#{entry.external_id}|#{entry.email}|#{entry.id})\n"
-      str << set_to_str(set, lev: lev + 1, format: format) unless !set || set.empty?
+  def parts_to_value(obj, parts)
+    parts.reduce(obj) do |object, part|
+      get_attr(object, part)
+    end
+  end
+
+  def get_attr(obj, part)
+    case
+    when !obj
+      nil
+    when part.is_a?(Symbol) || obj.respond_to?(part.to_sym)
+      obj.send(part.to_sym)
+    when part.start_with?(":")
+      get_attr(obj, part[1..-1])
+    when part.start_with?("details[")
+      if (obj.respond_to?(:details)) && details = obj.details
+        if match = part.match(/details\[(?<field>.*)\]/)
+          details[match[:field]]
+        else
+          raise "Review your -use-field expression. It should read: person.details[target-alt_id]"
+        end
+      end
+    when part.start_with?("account")
+      obj.account if obj.respond_to?(:account)
+    when part.start_with?("person")
+      obj
+    else
+      raise "Review your expression. Cannot recognize '#{part}' as part of '#{obj.class}'"
     end
   end