eco-helpers 2.0.19 → 2.0.21

data/lib/eco/api/session.rb

@@ -106,11 +106,13 @@ module Eco
       # @param attr [String] type (`Symbol`) or attribute (`String`) to target a specific parser.
       # @param source [Any] source value to be parsed.
       # @param phase [Symbol] the phase when this parser should be active.
-      def parse_attribute(attr, source, phase = :internal)
+      # @param phase [Symbol] the phase when this parser should be active.
+      # @return [Object] the parsed attribute.
+      def parse_attribute(attr, source, phase = :internal, deps: {})
         unless parsers = entry_factory.person_parser
           raise "There are no parsers defined"
         end
-        parsers.parse(attr, source, phase)
+        parsers.parse(attr, source, phase, deps: deps)
       end
 
       # @see Eco::API::Common::People::EntryFactory#export

data/lib/eco/api/usecases/default_cases.rb

@@ -13,6 +13,7 @@ require_relative 'default_cases/abstract_policygroup_abilities_case.rb'
 require_relative 'default_cases/analyse_people_case'
 require_relative 'default_cases/append_usergroups_case'
 require_relative 'default_cases/change_email_case'
+require_relative 'default_cases/clean_unknown_tags_case'
 require_relative 'default_cases/codes_to_tags_case'
 require_relative 'default_cases/create_case'
 require_relative 'default_cases/create_details_case'

data/lib/eco/api/usecases/default_cases/analyse_people_case.rb

@@ -5,41 +5,158 @@ class Eco::API::UseCases::DefaultCases::AnalysePeople < Eco::API::Common::Loader
   attr_reader :session, :people, :options
 
   def main(people, session, options, usecase)
+    options[:end_get] = false
     @session = session; @options = options; @people = people
 
-    save!(analysis)
+    case
+    when case_options[:identify_duplicates]
+      identify_duplicates
+    when case_options[:identify_unnamed]
+      identify_unnamed
+    else
+      session.logger.info("No analysis operation was specified")
+    end.tap do |people_involved|
+      if people_involved
+        to_csv(people_involved) if to_csv?
+        create_people_backup(people_involved) if results_people_backup?
+      end
+    end
   end
 
   private
 
-  def analysis
-    analytics.attribute = :name
-    analysed = analytics.analyse({
-      threshold: 0.20,
-      order:     [:average, :dice]
-    })
-    analytics.re_analyse(analysed, threshold: 0.5, order: [:average])
+  def identify_unnamed
+    similarity_analytics.unnamed.tap do |unnamed|
+      if unnamed.empty?
+        session.logger.info("There were no people with no name!!")
+      end
+    end
   end
 
-  def analytics
-    @analytics ||= people.analytics
+  def identify_duplicates
+    analysed = similarity_screening
+    if case_options[:ignore_matching_words]
+      puts "Fine tune results by ignoring matching words..."
+      analysed = strict_similarity(analysed)
+    end
+
+    similarity_analytics.newSimilarity(analysed).tap do |related_people|
+      if related_people.empty?
+        session.logger.info("There were no possible duplicates identified!!")
+      else
+        report = similarity_analytics.report(analysed, format: :txt)
+        save!(report)
+      end
+    end
   end
 
-  def file
-    @file ||= options.dig(:output, :file) || "analytics.txt"
+  def strict_similarity(analysed)
+    similarity_analytics.ignore_matching_words(analysed, **{
+      threshold: 0.5,
+      order: [:ngrams]
+    })
   end
 
-  def save!(analysed)
-    if analysed.empty?
-      session.logger.info("There were results identified!!")
-      return
+  def similarity_screening
+    similarity_analytics.attribute = field_similarity
+    options = {
+      threshold: 0.4,
+      order:     [:average, :dice]
+    }.tap do |opts|
+      opts.merge!(needle_read: facet_field_proc) if facet_field?
+      opts.merge!(unique_words: true) if unique_words?
+    end
+    analysed = similarity_analytics.analyse(**options)
+    puts "Got #{analysed.count} results after basic screening with #{options}"
+
+    return analysed if case_options[:only_screening]
+    options = {threshold: 0.5, order: [:average]}
+    puts "Going to rearrange results... with #{options}"
+    similarity_analytics.rearrange(analysed, **options).tap do |analysed|
+      puts "... got #{analysed.count} results after rearranging"
     end
+  end
+
+  def similarity_analytics
+    @analytics ||= people.similarity
+  end
+
+  def create_people_backup(cut = people, file = results_people_backup)
+    session.file_manager.save_json(cut, file)
+  end
+
+  def to_csv(data = people, file = csv_file)
+    opts = {}
+    opts.deep_merge!(export: {file: {name: file, format: :csv}})
+    opts.deep_merge!(export: {options: {nice_header: true}})
+    opts.deep_merge!(export: {options: {internal_names: true}})
+    #opts.deep_merge!(export: {options: {split_schemas: true}})
+    session.process_case("to-csv", type: :export, people: data, options: opts.merge(options.slice(:export)))
+  end
+
+  def unique_words?
+    case_options[:unique_words]
+  end
+
+  def field_similarity
+    return :name unless use_field?
+    use_field_proc
+  end
+
+  def use_field_proc
+    proc_value_access(use_field)
+  end
+
+  def facet_field_proc
+    proc_value_access(facet_field)
+  end
+
+  def use_field
+    case_options.dig(:use_field)
+  end
 
-    ext  = File.extname(file).downcase.delete(".")
+  def use_field?
+    !!use_field
+  end
+
+  def facet_field
+    case_options.dig(:facet_field)
+  end
+
+  def facet_field?
+    !!facet_field
+  end
+
+  def csv_file
+    case_options.dig(:csv_file)
+  end
+
+  def to_csv?
+    !!csv_file
+  end
+
+  def results_people_backup
+    case_options.dig(:backup_people)
+  end
+
+  def results_people_backup?
+    !!results_people_backup
+  end
+
+  def case_options
+    options.dig(:usecase, :analyse_people) || {}
+  end
 
-    File.open(file, "w") do |fd|
+  def output_file
+    @output_file ||= options.dig(:output, :file) || "analytics.txt"
+  end
+
+  def save!(data)
+    ext  = File.extname(output_file).downcase.delete(".")
+    session.logger.info("Generating file '#{output_file}'")
+    File.open(output_file, "w") do |fd|
       if ext == "txt"
-        fd << analytics.analysis(analysed, format: :txt)
+        fd << data
       elsif ext == "html"
         puts "html is still not supported"
         exit(1)
@@ -50,4 +167,57 @@ class Eco::API::UseCases::DefaultCases::AnalysePeople < Eco::API::Common::Loader
     end
   end
 
+  # A way to use command line to specify part
+  # => i.e. details[first-name] AND details[surname]
+  def proc_value_access(expression)
+    #return expression.to_sym if expression.start_with?(":")
+    subexpressions = expression.split(" AND ")
+    Proc.new do |person|
+      values = subexpressions.map {|exp| attribute_access(person, exp)}
+      values.compact.join(" ")
+    end
+  end
+
+  # A way to use command line to specify part
+  # => i.e. person.details[first-name]
+  def attribute_access(person, expression)
+    parts = expression.split(".")
+    parts_to_value(person, parts).tap do |value|
+      unless value.is_a?(String) || !value
+        raise "Something is wrong with #{expression} to parts #{parts}. Expecting String, obtained: #{value.class}"
+      end
+    end
+  end
+
+  def parts_to_value(obj, parts)
+    parts.reduce(obj) do |object, part|
+      get_attr(object, part)
+    end
+  end
+
+  def get_attr(obj, part)
+    case
+    when !obj
+      nil
+    when part.is_a?(Symbol) || obj.respond_to?(part.to_sym)
+      obj.send(part.to_sym)
+    when part.start_with?(":")
+      get_attr(obj, part[1..-1])
+    when part.start_with?("details[")
+      if (obj.respond_to?(:details)) && details = obj.details
+        if match = part.match(/details\[(?<field>.*)\]/)
+          details[match[:field]]
+        else
+          raise "Review your -use-field expression. It should read: person.details[target-alt_id]"
+        end
+      end
+    when part.start_with?("account")
+      obj.account if obj.respond_to?(:account)
+    when part.start_with?("person")
+      obj
+    else
+      raise "Review your expression. Cannot recognize '#{part}' as part of '#{obj.class}'"
+    end
+  end
+
 end

data/lib/eco/api/usecases/default_cases/clean_unknown_tags_case.rb

@@ -0,0 +1,37 @@
+class Eco::API::UseCases::DefaultCases::CleanUnknownTags < Eco::API::Common::Loaders::UseCase
+  name "clean-unknown-tags"
+  type :transform
+
+  REGISTER_TAGS = [
+    "EVENT", "INJURY", "RISK", "CONTRACTOR", "PERMIT",
+    "AUDIT", "JSEA",
+    "TRAINING", "INDUCTION",
+    "MEETING",  "PPE",  "CHEMICAL",
+    "PLANT", "ASSET",
+    "POLICY", "IDEA", "REPORTS"
+  ]
+
+  attr_reader :session, :options
+
+  def main(people, session, options, usecase)
+    @session = session; @options = options
+
+    update   = session.new_job("main", "update", :update, usecase)
+    people.each do |person|
+      unknown_tags = person.filter_tags.select {|tag| !tag?(tag)}
+      person.filter_tags -= unknown_tags
+      update.add(person)
+    end
+  end
+
+  private
+
+  def tag?(value)
+    tagtree.tag?(value) || REGISTER_TAGS.any? {|reg| value == reg}
+  end
+
+  def tagtree
+    @tagtree ||= ASSETS.config.tagtree
+  end
+
+end

data/lib/eco/cli/config/default/options.rb

@@ -18,6 +18,12 @@ ASSETS.cli.config do |cnf|
       exit
     end
 
+    desc = "Redirect Standard Ouput to file"
+    options_set.add("-stdout", desc) do |options, session|
+      file = SCR.get_arg("-stdout", with_param: true) || "output.txt"
+      STDOUT.reopen(file, "w+")
+    end
+
     desc = "Fix the current session to work with this schema"
     options_set.add("-schema-id", desc) do |options, session|
       sch_name = SCR.get_arg("-schema-id", with_param: true)
@@ -41,6 +47,18 @@ ASSETS.cli.config do |cnf|
       options.deep_merge!(input: {entries_from: true})
     end
 
+    desc  = "Used to only get the people from the input file. It will also include their current and new supervisors."
+    options_set.add("-get-partial", desc) do |options, session|
+      options.deep_merge!(people: {
+        get: {from: :remote, type: :partial}
+      })
+    end
+
+    desc  = "Do not load any people for this run."
+    options_set.add("-no-people", desc) do |options, session|
+      options.deep_merge!(people: {get: false})
+    end
+
     desc = "Locally cache all the people manager by retrieving from the server"
     options_set.add("-get-people", desc) do |options, session|
       options.deep_merge!(people: {
@@ -48,7 +66,17 @@ ASSETS.cli.config do |cnf|
       })
     end
 
-    options_set.add(["-dry-run", "-simulate"], "Runs in dry-run (no requests sent to server)") do |options, session|
+    desc  = "Used to specify the cache file of people to be used. "
+    desc += "It is useful to use as people reference those stored in cached file diffrent to the last one."
+    options_set.add("-people-from-backup", desc) do |options, session|
+      file = SCR.get_file("-people-from-backup", required: true, should_exist: true)
+      options.deep_merge!(people: {
+        get: {from: :local, type: :file, file: file}
+      })
+    end
+
+    desc  = "Runs in dry-run (no requests sent to server)"
+    options_set.add(["-dry-run", "-simulate"], desc) do |options, session|
       options[:dry_run]  = true
       options[:simulate] = true
       session.config.dry_run!

data/lib/eco/cli/config/default/people.rb

@@ -1,29 +1,23 @@
 ASSETS.cli.config do |cnf|
   cnf.people do |input, session, options|
-    people = Eco::API::Organization::People.new([])
-    if SCR.get_arg("-get-people")
-      options.deep_merge!(people: {
-        get: {from: :remote, type: :full}
-      })
-      people = session.micro.people_cache
-    elsif SCR.get_arg("-no-people")
-      people = Eco::API::Organization::People.new([])
-    elsif SCR.get_arg("-people-from-backup")
-      file = SCR.get_arg("-people-from-backup", with_param: true)
-      options.deep_merge!(people: {
-        get: {from: :local, type: :backup}
-      })
-      people = JSON.parse(File.read(file))
-      people = Eco::API::Organization::People.new(people)
-    elsif SCR.get_arg("-get-partial")
-      unless input && input.is_a?(Enumerable)
+    get = options.dig(:people, :get) || {}
+    case
+    when get == false
+      Eco::API::Organization::People.new([])
+    when (get[:from] == :remote) && get[:type] == :full
+      # -get-people
+      session.micro.people_cache
+    when (get[:from] == :remote) && get[:type] == :partial
+      # -get-partial
+      unless (input && input.is_a?(Enumerable))
         raise "To use -get-partial (partial updates), you need to use -entries-from"
       end
-      options.deep_merge!(people: {
-        get: {from: :remote, type: :partial}
-      })
-
-      people = session.micro.people_search(input, options: options)
+      session.micro.people_search(input, options: options)
+    when (get[:from] == :local)  && get[:type] == :file
+      # -people-from-backup
+      session.micro.people_load(get[:file], modifier: :file)
+      #people = JSON.parse(File.read(get[:file]))
+      #Eco::API::Organization::People.new(people)
     else
       options.deep_merge!(people: {
         get: {from: :local, type: :full}
@@ -33,9 +27,9 @@ ASSETS.cli.config do |cnf|
         options.deep_merge!(people: {
           get: {from: :remote, type: :full}
         })
-        people = session.micro.people_cache if people.empty?
+        people = session.micro.people_cache
       end
+      people
     end
-    people
   end
 end

data/lib/eco/cli/config/default/usecases.rb

@@ -26,11 +26,36 @@ ASSETS.cli.config do |cnf|
     end
 
     desc = "Provides a set of tools to analyse a set of people (i.e. detect duplicates)"
-    cases.add("-analyse-people", :export, desc, case_name: "-analyse-people") do |people, session, options|
+    cases.add("-analyse-people", :export, desc, case_name: "analyse-people") do |people, session, options|
       options.deep_merge!(output: {file: "people_analysis.txt"}) unless options.dig(:output, :file)
-    end.add_option("-to", "Specify the output file") do |options|
+      #unless options.dig(:usecase, :analyse_people, :use_field)
+      #  options.deep_merge!(usecase: {analyse_people: {use_field: :name}})
+      #end
+    end.add_option("-to", "Specify the output file.") do |options|
       file = SCR.get_file("-to", required: true, should_exist: false)
       options.deep_merge!(output: {file: file})
+    end.add_option("-identify-duplicates", "Generates a list of people with possible duplicates.") do |options|
+      options.deep_merge!(usecase: {analyse_people: {identify_duplicates: true}})
+    end.add_option("-use-field", "Works with -identify-duplicates. Sets field to be used in the comparison.") do |options|
+      expression = SCR.get_arg("-use-field", with_param: true)
+      options.deep_merge!(usecase: {analyse_people: {use_field: expression}})
+    end.add_option("-facet-field", "Works with -identify-duplicates. Adds an additional layer of comparison.") do |options|
+      expression = SCR.get_arg("-facet-field", with_param: true)
+      options.deep_merge!(usecase: {analyse_people: {facet_field: expression}})
+    end.add_option("-only-screening", "Works with -identify-duplicates. Skips the rearrangement stage.") do |options|
+      options.deep_merge!(usecase: {analyse_people: {only_screening: true}})
+    end.add_option("-ignore-matching-words", "Works with -identify-duplicates. Re-adjust scores ignoring matching words.") do |options|
+      options.deep_merge!(usecase: {analyse_people: {ignore_matching_words: true}})
+    end.add_option("-unique-words", "Works with -identify-duplicates. Re-adjust the comparing strings to do not have repeated words.") do |options|
+      options.deep_merge!(usecase: {analyse_people: {unique_words: true}})
+    end.add_option("-identify-unnamed", "Identifies all people with no names.") do |options|
+      options.deep_merge!(usecase: {analyse_people: {identify_unnamed: true}})
+    end.add_option("-backup-people-results", "Generates a json file with all the people involved in the final results of the analysis.") do |options|
+      file = SCR.get_file("-backup-people-results", required: true, should_exist: false)
+      options.deep_merge!(usecase: {analyse_people: {backup_people: File.expand_path(file)}})
+    end.add_option("-to-csv", "Genarates a CSV file with all people of the final results.") do |options|
+      file = SCR.get_file("-to-csv", required: true, should_exist: false) || "Results.csv"
+      options.deep_merge!(usecase: {analyse_people: {csv_file: File.expand_path(file)}})
     end
 
     desc = "It exports to a CSV the (filtered) people"
@@ -62,6 +87,10 @@ ASSETS.cli.config do |cnf|
       options.deep_merge!(other: {file: {codes_column: col_codes}})
     end
 
+    desc  = "Cleans from filter_tags those tags that are not present in the tagtree (as per tagtree.json file)."
+    desc += " It will preserve standard register tags of most common registers (i.e. EVENT, RISK)."
+    cases.add("-clean-unknown-tags", :transform, desc, case_name: "clean-unknown-tags")
+
     desc = "Removes the landing page or sets it to -page-id"
     cases.add("-reset-landing-page", :transform, desc, case_name: "reset-landing-page")
     .add_option("-page-id", "Target landing page to set to the users") do |options|