eco-helpers 2.0.19 → 2.0.25

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 14260868c76936513a93d4d104eacebbd11e47ed05806d4102ee76196a300d2b
-  data.tar.gz: 35784d03a18f89d2ce8bf5c4105e0eaa647dd10b4e1fee03897319d9ad838760
+  metadata.gz: 06a58306abadf9b27421583990eb14960f7f30368515481b16aa474de1bc1b08
+  data.tar.gz: 0eef93068fdb31bc6d1949f1022eac325403ac3dbb47c95b593a5b9623655773
 SHA512:
-  metadata.gz: 514d71e93bfa4fb854d9062be03306e154a4dfd184256ab03da30e2bf4bb2a45fb305efd5e2206821e522dc7cc0bfbc69e75285e3c6292dca78f359bd166f52a
-  data.tar.gz: c99a424905916cef61333c18bb31726e90fb9759a4a1b747b72ab123f4bc09ecc6962205a966b3ea46aa1f0a4cc3dd7f72a8c53829cc764a7993d64ad45495bf
+  metadata.gz: 80b0d2fc7bedb99deabae6d7d273cb4967eb0022db2e743078a82cace02d4f499fe8ad51ec02b7c5bcef549aac9fb03b0ea7ef5358fb602c65856654c7c20814
+  data.tar.gz: 553e1342f38c244ab57bb259b639d55ddc4a4d5d6f72bd54ed9290111636f4dffb29834f69a5b7d2707ee3d44951fa52efccf81589194f11dfa1a709309ddb77

data/CHANGELOG.md

@@ -1,7 +1,83 @@
 # Change Log
 All notable changes to this project will be documented in this file.
 
-## [2.0.19] - 2021-05-xx
+## [2.0.25] - 2021-06-xx
+
+### Added
+  - `Eco::API::UseCases::DefaultCases::HrisCase` validation error to require `-schema-id` command line when there are people in schemas other than the active one
+  
+### Changed
+  - `Eco::API::Session::Batch::Job`
+    * for backwards compatibility `-include-only-excluded` should bring an options structure compatible with `-include-excluded`
+
+### Fixed
+  - `Eco::API::Session::Batch` fixed typo that would prevent `prompt_user` to work
+
+## [2.0.24] - 2021-06-22
+
+### Added
+  - `Eco::API::Session::Batch::Job` made **native** `-include-excluded`
+    * also added new option `-include-only-excluded` to be able to only target people HRIS excluded
+
+
+## [2.0.23] - 2021-06-22
+
+### Added
+  - `Eco::API::Session::Batch::Job` **new** option (`-save-requests`) to save requests even if in `dry-run` (`-simulate`)
+### Changed
+  - `Eco::API::Session::Batch::Job` new people won't create updates unless they have either details or account
+    * because that entry is not supposed to be created unless has account or details
+
+## [2.0.22] - 2021-06-18
+
+### Added
+  - exposed `logger` in `BaseLoader` and
+  - support for multiple input files
+    * `Eco::API::Common::People::EntryFactory#entries`:
+      - refactored to allow multiple input files parsing
+      - moreover to `idx`, hash entries will get their `source_file`
+    * Input callback at `lib/eco/cli/config/default/input` refactored format detection and enabled folder input
+    * `SCR.get_file` language extended to also mention folder (not just file)
+  - support for `.xls` and `.xlsx` files
+    * `Eco::API::Common::People::DefaultParsers::XLSParser` the Excel files **parser**
+    * `Eco::API::Common::People::PersonParser` added `:xls` as an accepted format
+    * `Eco::API::Session#fields_mapper` exposed mapper through a method to allow **headers detection**
+      - The external names of the fields are the column headers of the input file
+    * `Eco::API::UseCases::BaseIO` when arguments validation rails, now it raises with specific `MissingParameter` error
+
+### Changed
+  - dry out `BaseLoader` (only session is set as instance variable)
+  - removed `creek` **dependency** (it was not used anywhere in the gem)
+    * we just kept `roo` and `roo-xls`
+  - custom `Error` classes now all inherit from `StandardError` (rather than `Exception`)
+
+
+
+## [2.0.21] - 2021-06-04
+
+### Added
+  - `Eco::CSV::Table`, support to create the table out of an `Array<Hash>`
+    - This opens new methods to transform input Excel file to this data structure and unify input data structures.
+  - **new** use case `Eco::API::UseCases::DefaultCases::CleanUnknownTags` invokable via `clean-unknown-tags`
+
+### Changed
+  - `Eco::API::Common::People::EntryFactory` slight **refactor** to boost better support for multiple input formats
+
+
+## [2.0.20] - 2021-05-31
+
+### Added
+  - **dependencies** to `creek`, `roo` and `roo-xls`
+  - **dependencies** to `hashdiff`
+  - `Eco::API::Session#parse_attribute` => added missing parameter `deps:`
+  - new option `-stdout [file]` to redirect the output to a file
+  - `Eco::CSV::Table`, **added** more helper methods `#group_by`, `#transform_values`, `#slice`, `#slice_columns`, `#delete_column`
+  - `Eco::API::Organization::TagTree` **added** more helper methods: `top?`, `tag=`, `as_json`, `dup`, `diff`
+
+### Fixed
+  - `Exception` patch: when `SystemStackError` there is not `backtrace` :/
+
+## [2.0.19] - 2021-05-31
 
 ### Added
   - Better error message for people searches & **offer** to select among the candidates:

data/eco-helpers.gemspec

@@ -14,7 +14,7 @@ Gem::Specification.new do |spec|
   spec.homepage      = "https://www.ecoportal.com"
   spec.licenses      = %w[MIT]
 
-  spec.required_ruby_version = '>= 2.4.4'
+  spec.required_ruby_version = '>= 2.5.0'
 
   spec.files         = `git ls-files -z`.split("\x0").reject do |f|
     f.match(%r{^(test|spec|features)/})
@@ -36,7 +36,10 @@ Gem::Specification.new do |spec|
   spec.add_dependency 'aws-sdk-ses',   '>= 1.36.0', '< 2'
   spec.add_dependency 'dotenv',        '>= 2.7.6', '< 2.8'
   spec.add_dependency 'net-sftp',      '>= 3.0.0', '< 3.1'
+  spec.add_dependency 'hashdiff',      '>= 1.0.1', '< 1.1'
   spec.add_dependency 'fuzzy_match',   '>= 2.1.0', '< 2.2'
   spec.add_dependency 'amatch',        '>= 0.4.0', '< 0.5'
   spec.add_dependency 'jaro_winkler',  '>= 1.5.4', '< 1.6'
+  spec.add_dependency 'roo',           '>= 2.8.3', '< 2.9'
+  spec.add_dependency 'roo-xls',       '>= 1.2.0', '< 1.3'
 end

data/lib/eco/api/common/base_loader.rb

@@ -51,15 +51,19 @@ module Eco
         private
 
         def session
-          @session ||= ASSETS.session
+          ASSETS.session
         end
 
-        def micro
-          session.micro
+        def config
+          session.config
         end
 
-        def config
-          @config ||= ASSETS.config
+        def logger
+          session.logger
+        end
+
+        def micro
+          session.micro
         end
 
       end

data/lib/eco/api/common/loaders/parser.rb

@@ -19,6 +19,7 @@ module Eco
               @attribute = value
             end
 
+            # TODO: it migh rather merge?
             # Some parsers require dependencies to do their job.
             def dependencies(value = nil)
               @dependencies ||= {}

data/lib/eco/api/common/people/default_parsers.rb

@@ -44,3 +44,4 @@ require_relative 'default_parsers/freemium_parser'
 require_relative 'default_parsers/policy_groups_parser'
 require_relative 'default_parsers/login_providers_parser'
 require_relative 'default_parsers/csv_parser'
+require_relative 'default_parsers/xls_parser'

data/lib/eco/api/common/people/default_parsers/xls_parser.rb

@@ -0,0 +1,53 @@
+class Eco::API::Common::People::DefaultParsers::XLSParser < Eco::API::Common::Loaders::Parser
+  attribute :xls
+
+  attr_accessor :already_required
+  attr_reader   :file
+
+  def parser(file, deps)
+    @file = file
+    rows.tap {|r| @file = nil}
+  end
+
+  def serializer(array_hash, deps)
+    raise "Not implemented. TODO: using axlsx or rubyXL gems. See: https://spin.atomicobject.com/2017/03/22/parsing-excel-files-ruby/"
+  end
+
+  private
+
+  def headers
+    raise "You should implement this method"
+  end
+
+  def sheet_name
+    0
+  end
+
+  def workbook
+    require_reading_libs!
+    Roo::Spreadsheet.open(file)
+  end
+
+  def spreadheet(name_or_index = sheet_name)
+    workbook.sheet(name_or_index)
+  end
+
+  def rows(target = headers)
+    begin
+      spreadheet.parse(header_search: target)
+    rescue Roo::HeaderRowNotFoundError => e
+      missing  = JSON.parse(e.message)
+      logger.warn("The input file is missing these headers: #{missing}")
+      present = target - missing
+      rows(present)
+    end
+  end
+
+  def require_reading_libs!
+    return if already_required
+    require 'roo'
+    require 'roo-xls'
+    already_required = true
+  end
+
+end

data/lib/eco/api/common/people/entries.rb

@@ -99,6 +99,7 @@ module Eco
             newFrom to_a - discarded
           end
 
+          # TODO: it should rather use the the people-to-csv case somehow
           # Helper to dump the entries into a CSV
           # @param filename [String] the destination file
           def export(filename)

data/lib/eco/api/common/people/entry_factory.rb

@@ -2,7 +2,7 @@ module Eco
   module API
     module Common
       module People
-        # TODO: EntryFactory should suppport multiple schemas itself
+        # TODO: EntryFactory should suppport multiple schemas itself (rather that being done on `Session`)
         # => currently, it's through session.entry_factory(schema: id), but this is wrong
         # => This way, Entries and PersonEntry will be able to refer to attr_map and person_parser linked to schema_id
         # => "schema_id" should be an optional column in the input file, or parsable via a custom parser to scope the schema
@@ -88,26 +88,63 @@ module Eco
             fatal("Format should be a Symbol. Given '#{format}'") if format && !format.is_a?(Symbol)
             fatal("There is no parser/serializer for format ':#{format.to_s}'") unless no_format || @person_parser.defined?(format)
 
-            if file
-              arr_hash = []
-              if Eco::API::Common::Session::FileManager.file_exists?(file)
-                encoding   ||= Eco::API::Common::Session::FileManager.encoding(file)
-                encoding     = (encoding != "utf-8")? "#{encoding}|utf-8": encoding
-                file_content = File.read(file, encoding: encoding)
-                arr_hash     = person_parser.parse(format, file_content).map.each_with_index do |entry_hash, i|
-                  j = (format == :csv)? i + 2 : i + 1
-                  entry_hash.tap {|hash| hash["idx"] = j}
-                end
+            kargs = {}
+            kargs.merge!(content:  data)     unless no_data
+            kargs.merge!(file:     file)     unless no_file
+            kargs.merge!(format:   format)   unless no_format
+            kargs.merge!(encoding: encoding) if encoding
+
+            Entries.new(to_array_of_hashes(**kargs), klass: PersonEntry, factory: self)
+          end
+
+          def to_array_of_hashes(**kargs)
+            data = []
+            content, file, encoding, format = kargs.values_at(:content, :file, :encoding, :format)
+
+            # Support for multiple file
+            if file.is_a?(Array)
+              return file.each_with_object([]) do |f, out|
+                logger.info("Parsing file '#{f}'")
+                curr = to_array_of_hashes(**kargs.merge(file: f))
+                out.concat(curr)
+              end
+            end
+            # Get content only when it's not :xls
+            # note: even if content was provided, file takes precedence
+            content = get_file_content(file, format, encoding) if (format != :xls) && file
+
+            case content
+            when Hash
+              logger.error("Input data as 'Hash' not supported. Expecting 'Enumerable' or 'String'")
+              exit(1)
+            when String
+              to_array_of_hashes(content: person_parser.parse(format, content))
+            when Enumerable
+              sample = content.to_a.first
+              case sample
+              when Hash, Array, ::CSV::Row
+                Eco::CSV::Table.new(content).to_array_of_hashes
               else
-                logger.warn("File does not exist: #{file}")
+                logger.error("Input content 'Array' of '#{sample.class}' is not supported.")
               end
-
-              entries(data: arr_hash)
             else
-              Entries.new(data, klass: PersonEntry, factory: self)
+              if file && format == :xls
+                person_parser.parse(format, file)
+              else
+                logger.error("Could not obtain any data out of these: #{kargs}. Given content: '#{content.class}'")
+                exit(1)
+              end
+            end.tap do |out_array|
+              start_from_two = (format == :csv) || format == :xls
+              out_array.each_with_index do |entry_hash, i|
+                entry_hash["idx"] = start_from_two ? i + 2 : i + 1
+                entry_hash["source_file"] = file
+              end
             end
+
           end
 
+
           # Helper that generates a file out of `data:`.
           # @raise Exception
           #   - if you try to provide `data:` in the wrong format.
@@ -127,7 +164,7 @@ module Eco
 
             run = true
             if Eco::API::Common::Session::FileManager.file_exists?(file)
-              prompt_user("The file '#{file}' already exists. Do you want to overwrite it? (Y/n):", default: "Y") do |response|
+              prompt_user("Do you want to overwrite it? (Y/n):", explanation: "The file '#{file}' already exists.", default: "Y") do |response|
                 run = (response == "") || reponse.upcase.start_with?("Y")
               end
             end
@@ -150,6 +187,17 @@ module Eco
 
           private
 
+          def get_file_content(file, format, encoding)
+            unless Eco::API::Common::Session::FileManager.file_exists?(file)
+              logger.error("File does not exist: #{file}")
+              exit(1)
+            end
+            ext        = File.extname(file)
+            encoding ||= Eco::API::Common::Session::FileManager.encoding(file)
+            encoding   = (encoding != "utf-8")? "#{encoding}|utf-8": encoding
+            content    = File.read(file, encoding: encoding)
+          end
+
           def fatal(msg)
             logger.fatal(msg)
             raise msg

data/lib/eco/api/common/people/person_parser.rb

@@ -16,7 +16,7 @@ module Eco
           CORE_ATTRS    = ["id", "external_id", "email", "name", "supervisor_id", "filter_tags", "freemium"]
           ACCOUNT_ATTRS = ["policy_group_ids", "default_tag", "send_invites", "landing_page_id", "login_provider_ids"]
           TYPE          = [:select, :text, :date, :number, :phone_number, :boolean, :multiple]
-          FORMAT        = [:csv, :xml, :json]
+          FORMAT        = [:csv, :xml, :json, :xls]
 
           attr_reader :schema
           attr_reader :details_attrs, :all_model_attrs

data/lib/eco/api/common/version_patches/exception.rb

@@ -2,8 +2,11 @@ class ::Exception
   def patch_full_message
     begin
       msg = []
-      msg << "\n#{backtrace.first} \n#{message} (#{self.class.to_s})"
-      backtrace[1..-1].each_with_index {|bt, i| msg << "#{" "*8}#{i+1}: from #{bt}"}
+      tracing   = backtrace ? backtrace : []
+      tracing   = (self.class == SystemStackError) ? tracing[1..30] : tracing[1..-1]
+      tracing ||= []
+      msg << "\n#{tracing.first} \n#{message} (#{self.class.to_s})"
+      tracing.each_with_index {|bt, i| msg << "#{" "*8}#{i+1}: from #{bt}"}
       msg.join("\n")
     rescue Exception => e
       puts "Something is wrong with 'patch_full_message': #{e}"

data/lib/eco/api/organization/people.rb

@@ -210,14 +210,20 @@ module Eco
           to_h(:supervisor_id)
         end
 
+        def group_by_schema
+          to_h do |person|
+            person.details && person.details.schema_id
+          end
+        end
+
         def to_h(attr = "id")
           super(attr || "id")
         end
         # @!endgroup
 
         # @!group Helper methods
-        def analytics
-          Eco::API::Organization::PeopleAnalytics.new(self.to_a)
+        def similarity
+          Eco::API::Organization::PeopleSimilarity.new(self.to_a)
         end
         # @!endgroup
 

data/lib/eco/api/organization/people_similarity.rb

@@ -13,7 +13,19 @@ module Eco
         # @!group Config
         # @return [String, Proc, nil] the target attribute to be read.
         def attribute=(attr)
-          @attribute ||= "name"
+          @attribute = attr
+        end
+
+        def attribute
+          @attribute ||= :name
+        end
+
+        # Returns the target value to analyse
+        # @param person [Ecoportal::API::V1::Person]
+        def item_value(person)
+          return attr.call(item) if attribute.is_a?(Proc)
+          attr = attribute.to_sym
+          return item.send(attr) if item.respond_to?(attr)
         end
 
         # Define the order or relevant of per user matches
@@ -37,6 +49,16 @@ module Eco
           @threshold ||= 0.15
         end
 
+        # Generates a new object with same config but different base `data`.
+        # @return [Eco::API::Organization::PeopleSimilarity]
+        def newFrom(data)
+          super(data).tap do |simil|
+            simil.threshold = threshold
+            simil.order     = order
+            simil.attribute = attribute
+          end
+        end
+
         # @!endgroup
 
         # @!group Searchers
@@ -50,36 +72,158 @@ module Eco
           end
         end
 
+        # It returns all people with no name
+        # @return [Eco::API::Organization::PeopleSimilarity]
+        def unnamed
+          select do |person|
+            person.name.to_s.strip.length < 2
+          end.yield_self do |results|
+            newFrom(results)
+          end
+        end
+
+        # It returns all people with no name
+        # @return [Eco::API::Organization::PeopleSimilarity]
+        def named
+          reject do |person|
+            person.name.to_s.strip.length < 2
+          end.yield_self do |results|
+            newFrom(results)
+          end
+        end
+
+        # It returns all the entries with `attribute` empty
+        # @return [Eco::API::Organization::PeopleSimilarity]
+        def blank_attribute
+          select do |person|
+            item_value(person).to_s.strip.length < 2
+          end.yield_self do |results|
+            newFrom(results)
+          end
+        end
+
+        # It returns all the entries with `attribute` **n0t** empty
+        # @return [Eco::API::Organization::PeopleSimilarity]
+        def attribute_present
+          reject do |person|
+            item_value(person).to_s.strip.length < 2
+          end.yield_self do |results|
+            newFrom(results)
+          end
+        end
+
         # @!endgroup
 
-        # @!group Analysers
+        # @!group Analisys starters
 
         # Analyses People bases on `options`
+        # @param needle_read [Proc, Symbol] when the value to read from `needle` object is different to the `:read` (`attribute`).
+        #   This allows to for example, facet `needle.name` (needle_read) against `haystack_item.details[alt_id]` (read).
+        # @param keep_empty [Boolean] to indicate if it should get rid of people with no results (based on threshold)
         # @return [Hash] where the _keys_ are the people `id`s and the _values_ the `Eco::Data::FuzzyMatch::Results`
-        def analyse(**options)
+        def analyse(needle_read: nil, keep_empty: false, **options)
           options = { read: self.attribute }.merge(options)
+          total = count; i = 1
           each_with_object({}) do |person, results|
-            results[person.id] = find_all_with_score(person, **options)
+            needle_str = needle_read ? item_string(person, needle_read) : nil
+            results[person.id] = find_all_with_score(person, needle_str: needle_str, **options)
+            print_progress("Analysed", total, i)
+            i += 1
+          end.yield_self do |analysed|
+            analysed = clean_empty(analysed) unless keep_empty
+            #puts "... #{analysed.count} results after cleaning empty"
+            analysed
+          end
+        end
+
+        # @!endgroup
+
+        # @!group Results Treatment
+
+        # Gets a new instance object of this class, with only people in results
+        # @param analysed [Hash] where the _keys_ are the people `id`s and _values_ the `Eco::Data::FuzzyMatch::Results`
+        # @return [Eco::API::Organization::PeopleSimilarity]
+        def newSimilarity(analysed)
+          newFrom(people_in_results(analysed))
+        end
+
+        def people_in_results(analysed)
+          analysed.each_with_object([]) do |(id, results), people|
+            related = results.each_with_object([self[id]]) do |result, related|
+              related << result.match
+            end
+            related.each {|person| people << person unless people.include?(person)}
           end
         end
 
+        # Removes from results those that do not have similar entries
+        def clean_empty(analysed)
+          analysed.select do |id, results|
+            !results.empty?
+          end
+        end
+
+        # Helper to do some treatment fo the results
+        # @param analysed [Hash] where the _keys_ are the people `id`s and _values_ the `Eco::Data::FuzzyMatch::Results`
+        # @return [Hash] where the _keys_ are the people `id`s and _values_ the `Eco::Data::FuzzyMatch::Results`
+        def with_analysed(analysed, keep_empty: false)
+          analysed.each_with_object({}) do |(id, results), reanalysed|
+            reanalysed[id] = yield(self[id], results)
+          end.yield_self do |reanalysed|
+            reanalysed = clean_empty(reanalysed) unless keep_empty
+            reanalysed
+          end.tap {|out| "with_analysed... returns #{out.count} records"}
+        end
+
         # Launches a reanalyis on `analysed` based on `options`
         # @param analysed [Hash] where the _keys_ are the people `id`s and the _values_ the `Eco::Data::FuzzyMatch::Results`
-        def re_analyse(analysed, **options)
-          analysed.each_with_object({}) do |(id, results), out|
-            out[id] = results.relevant_results(**options)
+        def rearrange(analysed, **options)
+          with_analysed(analysed) do |person, results|
+            results.relevant_results(**options)
           end
         end
 
-        # @!group Helpers
+        # Reanalyses by using a block to treat the needle and item values
+        def reanalyse(analysed, msg: "Reanalysing", **options, &block)
+          options = { read: self.attribute }.merge(options)
+          total = analysed.count; i = 1
+          with_analysed(analysed) do |person, results|
+            print_progress(msg, total, i)
+            i += 1
+            recalculate_results(results, &block)
+          end
+        end
+
+        # Renalyses by ignoring matching words between the `needle` and those found in `results`
+        def ignore_matching_words(analysed, **options)
+          prompt = "Reanalysing by ignoring matching words"
+          reanalyse(analysed, msg: prompt, **options) do |needle_str, item_str, needle, item|
+            self.class.remove_matching_words(needle_str, item_str)
+          end
+        end
+
+        # Renalyses by ignoring matching words between the `needle` and those found in `results`
+        def ignore_matching_words_old(analysed, **options)
+          options = { read: self.attribute }.merge(options)
+          total = analysed.count; i = 1
+          with_analysed(analysed) do |person, results|
+            print_progress("Reanalysing by ignoring matching words", total, i)
+            i += 1
+            ignore_same_words_score(results, **options)
+          end
+        end
+
+        # @!endgroup
+
+        # @!group Reporting Helpers
 
         # @return [String] well structured text
-        def analysis(analysed, format: :txt)
+        def report(analysed, format: :txt)
           case
           when format == :txt
             analysed.each_with_object("") do |(id, results), out|
               msg = results.results.map {|r| r.print}.join("\n  ")
-              "'#{self[id].identify}':\n  " + msg
+              out << "#{self[id].identify}:\n  " + msg + "\n"
             end
           end
         end
@@ -91,7 +235,7 @@ module Eco
         def print_analysis(**options)
           analysed = options[:analysed] || results_with_false_positives.analyse(**options)
           analysed.each_with_object({}) do |(id, results), out|
-            puts analysis(analysed)
+            puts report(analysed)
           end
         end
         # @!endgroup
@@ -105,6 +249,22 @@ module Eco
 
         private
 
+        def print_progress(msg, total, num)
+          return unless total > 10
+          puts "" unless num > 1
+          @print_msg_len ||= 0
+          percent = (100 * num.to_f / total).round(1)
+          msg     = "  #{msg}: #{percent}% (#{num} of #{total})\r"
+          @print_msg_len = msg.length unless @print_msg_len > msg.length
+          print msg
+          $stdout.flush
+          if percent > 99.9
+            sleep(0.2)
+            print "#{" " * @print_msg_len}\r"
+            $stdout.flush
+          end
+        end
+
 
       end
     end