eco-helpers 2.0.19 → 2.0.25

data/lib/eco/api/usecases/default_cases/clean_unknown_tags_case.rb

@@ -0,0 +1,37 @@
+class Eco::API::UseCases::DefaultCases::CleanUnknownTags < Eco::API::Common::Loaders::UseCase
+  name "clean-unknown-tags"
+  type :transform
+
+  REGISTER_TAGS = [
+    "EVENT", "INJURY", "RISK", "CONTRACTOR", "PERMIT",
+    "AUDIT", "JSEA",
+    "TRAINING", "INDUCTION",
+    "MEETING",  "PPE",  "CHEMICAL",
+    "PLANT", "ASSET",
+    "POLICY", "IDEA", "REPORTS"
+  ]
+
+  attr_reader :session, :options
+
+  def main(people, session, options, usecase)
+    @session = session; @options = options
+
+    update   = session.new_job("main", "update", :update, usecase)
+    people.each do |person|
+      unknown_tags = person.filter_tags.select {|tag| !tag?(tag)}
+      person.filter_tags -= unknown_tags
+      update.add(person)
+    end
+  end
+
+  private
+
+  def tag?(value)
+    tagtree.tag?(value) || REGISTER_TAGS.any? {|reg| value == reg}
+  end
+
+  def tagtree
+    @tagtree ||= ASSETS.config.tagtree
+  end
+
+end

data/lib/eco/api/usecases/default_cases/hris_case.rb

@@ -3,8 +3,11 @@ class Eco::API::UseCases::DefaultCases::HrisCase < Eco::API::Common::Loaders::Us
   type :sync
 
   attr_reader :creation, :update, :supers, :leavers
+  attr_reader :people, :session, :options
 
   def main(entries, people, session, options, usecase)
+    @session = session; @options = options; @people = people
+    require_only_one_schema!
     micro    = session.micro
     @creation = session.new_job("main", "create",  :create, usecase)
     @update   = session.new_job("main", "update",  :update, usecase)
@@ -30,4 +33,21 @@ class Eco::API::UseCases::DefaultCases::HrisCase < Eco::API::Common::Loaders::Us
     person.account       = nil if person.account
   end
 
+  def require_only_one_schema!
+    unless schema_id = options.dig(:people, :filter, :details, :schema_id)
+      active_schema = session.schema
+      other_schemas = session.schemas.map(&:id) - [active_schema.id]
+      other_people  = people.group_by_schema.values_at(*other_schemas).map(&:to_a).flatten
+      if other_people.length > 3
+        msg  = "There are #{other_people.length} people in schemas other than #{active_schema.name}."
+        msg << " Please, use the filter option '-schema_id SchemaName' for the 'hris' case to only include those of that schema"
+        msg << " in the current update. The HRIS case identifies people that are not in the file as leavers."
+        msg << " (as it will remove the account of all the people of other schemas if they are not in the input file)."
+        msg << "\n For example: -schema-id '#{active_schema.name.downcase}'"
+        logger.error(msg)
+        raise msg
+      end
+    end
+  end
+
 end

data/lib/eco/cli/config/default/input.rb

@@ -1,18 +1,71 @@
 ASSETS.cli.config do |cnf|
+  formats = {
+    csv: {
+      option: ["-csv"],
+      extname: [".csv", ".txt"]
+    },
+    xml: {
+      option: ["-xml"],
+      extname: [".xml"]
+    },
+    xls:  {
+      option: ["-xls", "-xlsx", "-excel"],
+      extname: [".xls", ".xlsx", ".xlsm"]
+    },
+    json: {
+      option: ["-json"],
+      extname: [".json"]
+    }
+  }
+
   cnf.input(default_option: "-entries-from") do |session, str_opt, options|
     input = []
     if SCR.get_arg(str_opt)
       file = SCR.get_file(str_opt, required: true)
+
+      # Command line check
+      format = formats.reduce(nil) do |matched, (format, selectors)|
+        used = selectors[:option].reduce(false) {|used, option| SCR.get_arg(option) || used}
+        next matched if matched
+        next format  if used
+      end
+
+      # File/Folder check
+      file = File.expand_path(file)
+      if File.directory?(file)
+        folder = file
+        file   = Dir.glob("#{file}/*").reject {|f| File.directory?(f)}
+        ext    = (format && formats[format][:extname]) || [File.extname(file.first)]
+        file   = file.select {|f| ext.any? {|e| File.extname(f) == e}}.tap do |files|
+          if files.empty?
+            session.logger.error("Could not find any file with extension: #{ext} in folder '#{folder}'")
+            exit(1)
+          end
+        end
+      else
+        ext    = File.extname(file)
+      end
+
+      format ||= formats.reduce(nil) do |matched, (format, selectors)|
+        next matched if matched
+        next format  if selectors[:extname].any? {|e| ext == e}
+      end
+      format ||= :csv
+
       options.deep_merge!(input: {file: {name: file}})
-      if SCR.get_arg("-xml")
-        options.deep_merge!(input: {file: {format: :xml}})
-        session.config.files.validate(:xml, file)
-        input = session.entries(file: file, format: :xml)
-      elsif SCR.get_arg("-json")
-        options.deep_merge!(input: {file: {format: :json}})
-        input = Eco::API::Organization::People.new(JSON.parse(File.read(file)))
+      options.deep_merge!(input: {file: {format: format}})
+
+      case format
+      when :xml
+        [file].flatten.each {|f| session.config.files.validate(:xml, f)}
+        input = session.entries(file: file, format: format)
+      when :xls
+        input = session.entries(file: file, format: format)
+      when :json
+        input = [file].flatten.reduce(Eco::API::Organization::People.new([])) do |people, file|
+          people.merge(JSON.parse(File.read(file)))
+        end
       else
-        options.deep_merge!(input: {file: {format: :csv}})
         input = session.csv_entries(file)
       end
     end

data/lib/eco/cli/config/default/options.rb

@@ -18,6 +18,12 @@ ASSETS.cli.config do |cnf|
       exit
     end
 
+    desc = "Redirect Standard Ouput to file"
+    options_set.add("-stdout", desc) do |options, session|
+      file = SCR.get_arg("-stdout", with_param: true) || "output.txt"
+      STDOUT.reopen(file, "w+")
+    end
+
     desc = "Fix the current session to work with this schema"
     options_set.add("-schema-id", desc) do |options, session|
       sch_name = SCR.get_arg("-schema-id", with_param: true)
@@ -35,12 +41,24 @@ ASSETS.cli.config do |cnf|
       session.schema = sch_id
     end
 
-    desc  = "Used to be used to specify the input file when using -get-partial. "
+    desc  = "Used to be used to specify the input file or folder when using -get-partial."
     desc += "It can also be useful to obtain `-get-partial` of people base on `:export` use cases (i.e. -people-to-csv)"
     options_set.add("-entries-from", desc) do |options, session|
       options.deep_merge!(input: {entries_from: true})
     end
 
+    desc  = "Used to only get the people from the input file. It will also include their current and new supervisors."
+    options_set.add("-get-partial", desc) do |options, session|
+      options.deep_merge!(people: {
+        get: {from: :remote, type: :partial}
+      })
+    end
+
+    desc  = "Do not load any people for this run."
+    options_set.add("-no-people", desc) do |options, session|
+      options.deep_merge!(people: {get: false})
+    end
+
     desc = "Locally cache all the people manager by retrieving from the server"
     options_set.add("-get-people", desc) do |options, session|
       options.deep_merge!(people: {
@@ -48,12 +66,38 @@ ASSETS.cli.config do |cnf|
       })
     end
 
-    options_set.add(["-dry-run", "-simulate"], "Runs in dry-run (no requests sent to server)") do |options, session|
+    desc = "Saves the requests's body even though running in dry-run (-simulate)"
+    options_set.add("-save-requests", desc) do |options, session|
+      options.deep_merge!(requests: {backup: true})
+    end
+
+    desc  = "Used to specify the cache file of people to be used. "
+    desc += "It is useful to use as people reference those stored in cached file diffrent to the last one."
+    options_set.add("-people-from-backup", desc) do |options, session|
+      file = SCR.get_file("-people-from-backup", required: true, should_exist: true)
+      options.deep_merge!(people: {
+        get: {from: :local, type: :file, file: file}
+      })
+    end
+
+    desc  = "Runs in dry-run (no requests sent to server)"
+    options_set.add(["-dry-run", "-simulate"], desc) do |options, session|
       options[:dry_run]  = true
       options[:simulate] = true
       session.config.dry_run!
     end
 
+    desc  = "(careful with this option)  This will include everybody as part of the update (including those that are api excluded). "
+    desc += "Only launch with this option when only api excluded people are included in your update."
+    options_set.add("-include-excluded", desc) do |options|
+      options.deep_merge!(include: {excluded: true})
+    end
+
+    desc = "Includes in API updates ONLY people that evaluate true as people excluded from periodic upates."
+    options_set.add("-include-only-excluded", desc) do |options|
+      options.deep_merge!(include: {excluded: {only: true}})
+    end
+
     desc = "Ignores threshold limitations on requests for this session (skip batch belt)"
     options_set.add("-skip-batch-policy", desc) do |options|
       options.deep_merge!(skip: {batch_policy: true})

data/lib/eco/cli/config/default/people.rb

@@ -1,29 +1,23 @@
 ASSETS.cli.config do |cnf|
   cnf.people do |input, session, options|
-    people = Eco::API::Organization::People.new([])
-    if SCR.get_arg("-get-people")
-      options.deep_merge!(people: {
-        get: {from: :remote, type: :full}
-      })
-      people = session.micro.people_cache
-    elsif SCR.get_arg("-no-people")
-      people = Eco::API::Organization::People.new([])
-    elsif SCR.get_arg("-people-from-backup")
-      file = SCR.get_arg("-people-from-backup", with_param: true)
-      options.deep_merge!(people: {
-        get: {from: :local, type: :backup}
-      })
-      people = JSON.parse(File.read(file))
-      people = Eco::API::Organization::People.new(people)
-    elsif SCR.get_arg("-get-partial")
-      unless input && input.is_a?(Enumerable)
+    get = options.dig(:people, :get) || {}
+    case
+    when get == false
+      Eco::API::Organization::People.new([])
+    when (get[:from] == :remote) && get[:type] == :full
+      # -get-people
+      session.micro.people_cache
+    when (get[:from] == :remote) && get[:type] == :partial
+      # -get-partial
+      unless (input && input.is_a?(Enumerable))
         raise "To use -get-partial (partial updates), you need to use -entries-from"
       end
-      options.deep_merge!(people: {
-        get: {from: :remote, type: :partial}
-      })
-
-      people = session.micro.people_search(input, options: options)
+      session.micro.people_search(input, options: options)
+    when (get[:from] == :local)  && get[:type] == :file
+      # -people-from-backup
+      session.micro.people_load(get[:file], modifier: :file)
+      #people = JSON.parse(File.read(get[:file]))
+      #Eco::API::Organization::People.new(people)
     else
       options.deep_merge!(people: {
         get: {from: :local, type: :full}
@@ -33,9 +27,9 @@ ASSETS.cli.config do |cnf|
         options.deep_merge!(people: {
           get: {from: :remote, type: :full}
         })
-        people = session.micro.people_cache if people.empty?
+        people = session.micro.people_cache
       end
+      people
     end
-    people
   end
 end

data/lib/eco/cli/config/default/usecases.rb

@@ -26,11 +26,36 @@ ASSETS.cli.config do |cnf|
     end
 
     desc = "Provides a set of tools to analyse a set of people (i.e. detect duplicates)"
-    cases.add("-analyse-people", :export, desc, case_name: "-analyse-people") do |people, session, options|
+    cases.add("-analyse-people", :export, desc, case_name: "analyse-people") do |people, session, options|
       options.deep_merge!(output: {file: "people_analysis.txt"}) unless options.dig(:output, :file)
-    end.add_option("-to", "Specify the output file") do |options|
+      #unless options.dig(:usecase, :analyse_people, :use_field)
+      #  options.deep_merge!(usecase: {analyse_people: {use_field: :name}})
+      #end
+    end.add_option("-to", "Specify the output file.") do |options|
       file = SCR.get_file("-to", required: true, should_exist: false)
       options.deep_merge!(output: {file: file})
+    end.add_option("-identify-duplicates", "Generates a list of people with possible duplicates.") do |options|
+      options.deep_merge!(usecase: {analyse_people: {identify_duplicates: true}})
+    end.add_option("-use-field", "Works with -identify-duplicates. Sets field to be used in the comparison.") do |options|
+      expression = SCR.get_arg("-use-field", with_param: true)
+      options.deep_merge!(usecase: {analyse_people: {use_field: expression}})
+    end.add_option("-facet-field", "Works with -identify-duplicates. Adds an additional layer of comparison.") do |options|
+      expression = SCR.get_arg("-facet-field", with_param: true)
+      options.deep_merge!(usecase: {analyse_people: {facet_field: expression}})
+    end.add_option("-only-screening", "Works with -identify-duplicates. Skips the rearrangement stage.") do |options|
+      options.deep_merge!(usecase: {analyse_people: {only_screening: true}})
+    end.add_option("-ignore-matching-words", "Works with -identify-duplicates. Re-adjust scores ignoring matching words.") do |options|
+      options.deep_merge!(usecase: {analyse_people: {ignore_matching_words: true}})
+    end.add_option("-unique-words", "Works with -identify-duplicates. Re-adjust the comparing strings to do not have repeated words.") do |options|
+      options.deep_merge!(usecase: {analyse_people: {unique_words: true}})
+    end.add_option("-identify-unnamed", "Identifies all people with no names.") do |options|
+      options.deep_merge!(usecase: {analyse_people: {identify_unnamed: true}})
+    end.add_option("-backup-people-results", "Generates a json file with all the people involved in the final results of the analysis.") do |options|
+      file = SCR.get_file("-backup-people-results", required: true, should_exist: false)
+      options.deep_merge!(usecase: {analyse_people: {backup_people: File.expand_path(file)}})
+    end.add_option("-to-csv", "Genarates a CSV file with all people of the final results.") do |options|
+      file = SCR.get_file("-to-csv", required: true, should_exist: false) || "Results.csv"
+      options.deep_merge!(usecase: {analyse_people: {csv_file: File.expand_path(file)}})
     end
 
     desc = "It exports to a CSV the (filtered) people"
@@ -62,6 +87,10 @@ ASSETS.cli.config do |cnf|
       options.deep_merge!(other: {file: {codes_column: col_codes}})
     end
 
+    desc  = "Cleans from filter_tags those tags that are not present in the tagtree (as per tagtree.json file)."
+    desc += " It will preserve standard register tags of most common registers (i.e. EVENT, RISK)."
+    cases.add("-clean-unknown-tags", :transform, desc, case_name: "clean-unknown-tags")
+
     desc = "Removes the landing page or sets it to -page-id"
     cases.add("-reset-landing-page", :transform, desc, case_name: "reset-landing-page")
     .add_option("-page-id", "Target landing page to set to the users") do |options|

data/lib/eco/cli/config/default/workflow.rb

@@ -53,8 +53,7 @@ ASSETS.cli.config do |config|
           cases_with_people = config.usecases.active(io: io).select do |usecase, data|
             io.class.people_required?(usecase.type)
           end
-          get_people = io.options.dig(:people, :get, :from) == :remote
-          next io unless !cases_with_people.empty? || get_people
+          next io if cases_with_people.empty? && !io.options.dig(:people, :get)
           io = io.new(people:  config.people(io: io))
         end
 
@@ -67,7 +66,8 @@ ASSETS.cli.config do |config|
 
     wf.before(:usecases) do |wf_cases, io|
       # save partial entries -> should be native to session.workflow
-      partial_update = io.options.dig(:people, :get, :type) == :partial
+      get_people     = io.options.dig(:people, :get)
+      partial_update = get_people && get_people.dig(:type) == :partial
       if !io.options[:dry_run] && partial_update
         partial_file = io.session.config.people.partial_cache
         io.session.file_manager.save_json(io.people, partial_file, :timestamp)
@@ -98,11 +98,12 @@ ASSETS.cli.config do |config|
       if io.session.post_launch.empty?
         wf_post.skip!
       else
-        partial_update = io.options.dig(:people, :get, :type) == :partial
+        get_people     = io.options.dig(:people, :get)
+        partial_update = get_people && get_people.dig(:type) == :partial
         if !io.options[:dry_run] && partial_update
           # get target people afresh
           people = io.session.micro.people_refresh(people: io.people, include_created: true)
-          io     = io.new(people: people)
+          io     = io.base.new(people: people)
         else
           wf_post.skip!
           msg = "Although there are post_launch cases, they will NOT be RUN"
@@ -139,7 +140,8 @@ ASSETS.cli.config do |config|
     end
 
     wf.on(:end) do |wf_end, io|
-      partial_update = io.options.dig(:people, :get, :type) == :partial
+      get_people     = io.options.dig(:people, :get)
+      partial_update = get_people && get_people.dig(:type) == :partial
       unless !io.options[:end_get] || io.options[:dry_run] || partial_update
         people = io.session.micro.people_cache
         io     = io.new(people: people)

data/lib/eco/cli/scripting/args_helpers.rb

@@ -75,10 +75,10 @@ module Eco
         def get_file(key, required: false, should_exist: true)
           filename = get_arg(key, with_param: true)
           if !filename && required
-            puts "You need to specify a file '#{key} file'"
+            puts "You need to specify a file or folder '#{key} file_or_folder'"
             exit(1)
           elsif !file_exists?(filename) && should_exist && required
-            puts "This file doesn't exist '#{filename}'"
+            puts "This file/folder doesn't exist '#{filename}'"
             exit(1)
           end
 

data/lib/eco/csv/table.rb

@@ -1,4 +1,3 @@
-
 module Eco
   class CSV
     class Table < ::CSV::Table
@@ -9,6 +8,70 @@ module Eco
         super(to_rows_array(input))
       end
 
+      # @return [Hash] where keys are the groups and the values a `Eco::CSV::Table`
+      def group_by(&block)
+        rows.group_by(&block).transform_values do |rows|
+          self.class.new(rows)
+        end
+      end
+
+      # @return [Eco::CSV::Table]
+      def transform_values
+        transformed_rows = rows.map do |row|
+          res = yield(row)
+          case res
+          when Array
+            ::CSV::Row.new(row.headers, res)
+          when ::CSV::Row
+            res
+          end
+        end
+        self.class.new(transformed_rows)
+      end
+
+      # Slices the selected rows
+      # @return [Eco::CSV::Table]
+      def slice(*index)
+        case index.first
+        when Range, Numeric
+          self.class.new(rows.slice(index.first))
+        else
+          self
+        end
+      end
+
+      # @return [Eco::CSV::Table]
+      def slice_columns(*index)
+        case index.first
+        when Range, Numeric
+          columns_to_table(columns.slice(index.first))
+        when String
+          csv_cols = columns
+          csv_cols = index.each_with_object([]) do |name, cols|
+            col = csv_cols.find {|col| col.first == name}
+            cols << col if col
+          end
+          columns_to_table(csv_cols)
+        else
+          self
+        end
+      end
+
+      # @return [Eco::CSV::Table]
+      def delete_column(i)
+        csv_cols = columns
+        csv_cols.delete(i)
+        columns_to_table(csv_cols)
+      end
+
+      # Adds a new column at the end
+      # @param header_name [String] header of the new column
+      # @return [Eco::CSV::Table] with a new empty column
+      def add_column(header_name)
+        new_col = Array.new(length).unshift(header_name)
+        columns_to_table(columns.push(new_col))
+      end
+
       # @return [Array<::CSV::Row>]
       def rows
         [].tap do |out|
@@ -16,24 +79,40 @@ module Eco
         end
       end
 
+      # It removes all rows where all columns' values are the same
+      def delete_duplicates!
+        unique_rows = []
+        self.by_row!.delete_if do |row|
+          unique_rows.any? {|done| equal_rows?(row, done)}.tap do |found|
+            unique_rows << row unless found
+          end
+        end
+      end
+
+      # @param row1 [CSV:Row] row to be compared
+      # @param row2 [CSV:Row] row to be compared
+      # @param [Boolean] `true` if all values of `row1` are as of `row2`
+      def equal_rows?(row1, row2)
+        row1.fields.zip(row2.fields).all? do |(v1, v2)|
+          v1 == v2
+        end
+      end
+
       # @return [Integer] total number of rows not including the header
       def length
         to_a.length - 1
       end
 
+      def empty?
+        length < 1
+      end
+
       # @return [Array<Array>] each array is the column header followed by its values
       def columns
         to_a.transpose
       end
 
-      # Adds a new column at the end
-      # @param header_name [String] header of the new column
-      # @return [Eco::CSV::Table] with a new empty column
-      def add_column(header_name)
-        new_col = Array.new(length).unshift(header_name)
-        columns_to_table(columns.push(new_col))
-      end
-
+      # Creates a single `Hash` where each key, value is a column (header + values)
       # @note it will override columns with same header name
       # @return [Hash] keys are headers, values are arrays
       def columns_hash
@@ -42,6 +121,17 @@ module Eco
         end.to_h
       end
 
+      # Returns an array of row hashes
+      # @note it will override columns with same header
+      def to_a_h
+        rows.map(&:to_h)
+      end
+
+      # @see #to_a_h
+      def to_array_of_hashes
+        to_a_h
+      end
+
       private
 
       def columns_to_table(columns_array)
@@ -51,24 +141,34 @@ module Eco
 
       def to_rows_array(data)
         case data
-        when Array
-          return data unless data.length > 0
-          if data.first.is_a?(::CSV::Row)
-            data
-          elsif data.first.is_a?(Array)
-            headers  = data.shift
-            data.map do |arr_row|
-              CSV::Row.new(headers, arr_row)
-            end.compact
-          else
-            raise "Expected data that can be transformed into Array<Array>"
-          end
         when ::CSV::Table
           to_rows_array(data.to_a)
         when Hash
           # hash of columns header as key and column array as value
           rows_arrays = [a.keys].concat(a.values.first.zip(*a.values[1..-1]))
           to_rows_array(data.keys)
+        when Enumerable
+          data = data.dup.compact
+          return data unless data.count > 0
+          sample = data.first
+
+          case sample
+          when ::CSV::Row
+            data
+          when Array
+            headers  = data.shift
+            data.map do |arr_row|
+              ::CSV::Row.new(headers, arr_row)
+            end.compact
+          when Hash
+            headers     = sample.keys
+            headers_str = headers.map(&:to_s)
+            data.map do |hash|
+              ::CSV::Row.new(headers_str, hash.values_at(*headers))
+            end.compact
+          else
+            raise "Expected data that can be transformed into Array<::CSV::Row>. Given 'Enumerable' of '#{sample.class}'"
+          end
         else
           raise "Input type not supported. Given: #{data.class}"
         end