eco-helpers 2.0.18 → 2.0.24

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 678fc1c610a5aafd8321a74367a6aaca67aa70bf159f638209b45da0361bb575
-  data.tar.gz: 280a74a3a3877c5d58bfbe84d0f6a8f8b6f8eda5c68f376fea87522b03a59e42
+  metadata.gz: a2d943cd8ff8df507a2285548d7bf0c8fffa4009df45de1c26f38e4c966c747f
+  data.tar.gz: 14cdb53085deec51b406000048aa00199478ef9458290061cb47d324957ec1f2
 SHA512:
-  metadata.gz: 30c74c7aa8b7f59a4610cb4da804714a165a095d6280b756dcec17df37a81f2b5438a7178cc279750e5f97a43ae6d0041e464d9ed7965af39c39fc71ce4efc99
-  data.tar.gz: 1abe7f74b0a0452565c1add7778086030372f63912fe2c6279a2d40eb9f908d07746421507241d4905bbdf546e944899fa0f63fa3e3fe086caf1663c83a2624d
+  metadata.gz: 8563f5eaac3f8b650b62138fd31252d990467be498f9620bf7276a430ab8470acf1f2a8ab7b3e6ebcb9accedc1af94ac7168e6acdf3eb94de516259069a49c80
+  data.tar.gz: 881d6742b555d4b3cf42addef76470b35764684cbca0f903252ae5a39f503dc6a89d9bd7f7f774c587c0b6a23a6f898274af227baf14090d40ae30a16144eecc

data/CHANGELOG.md

@@ -1,13 +1,92 @@
 # Change Log
 All notable changes to this project will be documented in this file.
 
+## [2.0.24] - 2021-06-xx
+
+### Added
+  - `Eco::API::Session::Batch::Job` made **native** `-include-excluded`
+    * also added new option `-include-only-excluded` to be able to only target people HRIS excluded
+### Changed
+### Fixed
+
+
+## [2.0.23] - 2021-06-22
+
+### Added
+  - `Eco::API::Session::Batch::Job` **new** option (`-save-requests`) to save requests even if in `dry-run` (`-simulate`)
+### Changed
+  - `Eco::API::Session::Batch::Job` new people won't create updates unless they have either details or account
+    * because that entry is not supposed to be created unless has account or details
+
+## [2.0.22] - 2021-06-18
+
+### Added
+  - exposed `logger` in `BaseLoader` and
+  - support for multiple input files
+    * `Eco::API::Common::People::EntryFactory#entries`:
+      - refactored to allow multiple input files parsing
+      - moreover to `idx`, hash entries will get their `source_file`
+    * Input callback at `lib/eco/cli/config/default/input` refactored format detection and enabled folder input
+    * `SCR.get_file` language extended to also mention folder (not just file)
+  - support for `.xls` and `.xlsx` files
+    * `Eco::API::Common::People::DefaultParsers::XLSParser` the Excel files **parser**
+    * `Eco::API::Common::People::PersonParser` added `:xls` as an accepted format
+    * `Eco::API::Session#fields_mapper` exposed mapper through a method to allow **headers detection**
+      - The external names of the fields are the column headers of the input file
+    * `Eco::API::UseCases::BaseIO` when arguments validation rails, now it raises with specific `MissingParameter` error
+
+### Changed
+  - dry out `BaseLoader` (only session is set as instance variable)
+  - removed `creek` **dependency** (it was not used anywhere in the gem)
+    * we just kept `roo` and `roo-xls`
+  - custom `Error` classes now all inherit from `StandardError` (rather than `Exception`)
+
+
+
+## [2.0.21] - 2021-06-04
+
+### Added
+  - `Eco::CSV::Table`, support to create the table out of an `Array<Hash>`
+    - This opens new methods to transform input Excel file to this data structure and unify input data structures.
+  - **new** use case `Eco::API::UseCases::DefaultCases::CleanUnknownTags` invokable via `clean-unknown-tags`
+
+### Changed
+  - `Eco::API::Common::People::EntryFactory` slight **refactor** to boost better support for multiple input formats
+
+
+## [2.0.20] - 2021-05-31
+
+### Added
+  - **dependencies** to `creek`, `roo` and `roo-xls`
+  - **dependencies** to `hashdiff`
+  - `Eco::API::Session#parse_attribute` => added missing parameter `deps:`
+  - new option `-stdout [file]` to redirect the output to a file
+  - `Eco::CSV::Table`, **added** more helper methods `#group_by`, `#transform_values`, `#slice`, `#slice_columns`, `#delete_column`
+  - `Eco::API::Organization::TagTree` **added** more helper methods: `top?`, `tag=`, `as_json`, `dup`, `diff`
+
+### Fixed
+  - `Exception` patch: when `SystemStackError` there is not `backtrace` :/
+
+## [2.0.19] - 2021-05-31
+
+### Added
+  - Better error message for people searches & **offer** to select among the candidates:
+    - `Eco::API::Organization::People::MultipleSearchResults`, triggered from `Eco::API::Organization::People#find`
+    - `Eco::API::MicroCases#with_each` will offer the selection of candidates
+
+### Changed
+  - **renamed** and repurposed `Eco::API::Organization::PeopleAnalytics` to `PeopleSimilarity`
+
+### Fixed
+  - `Eco::Data::FuzzyMatch` adjustments for configuration propagation + some fixes
+  - Command option `-entries-from` can still be useful when used to obtain `-get-partial` of people base for `:export` use cases !!
+
 ## [2.0.18] - 2021-05-25
 
 ### Added
   - **`-one-off`** option to not having to type the `-api-key` every time you launch one-off scripts
     - `-api-key` will store the key to the `./.env_one_off` file (supports update and multi-environment)
 
-### Changed
 ### Fixed
   - patched `Exception#patch_full_message` to do not enter into a cyclic error rescue
   - also rescue on `workflow.rescue`

data/eco-helpers.gemspec

@@ -14,7 +14,7 @@ Gem::Specification.new do |spec|
   spec.homepage      = "https://www.ecoportal.com"
   spec.licenses      = %w[MIT]
 
-  spec.required_ruby_version = '>= 2.4.4'
+  spec.required_ruby_version = '>= 2.5.0'
 
   spec.files         = `git ls-files -z`.split("\x0").reject do |f|
     f.match(%r{^(test|spec|features)/})
@@ -36,7 +36,10 @@ Gem::Specification.new do |spec|
   spec.add_dependency 'aws-sdk-ses',   '>= 1.36.0', '< 2'
   spec.add_dependency 'dotenv',        '>= 2.7.6', '< 2.8'
   spec.add_dependency 'net-sftp',      '>= 3.0.0', '< 3.1'
+  spec.add_dependency 'hashdiff',      '>= 1.0.1', '< 1.1'
   spec.add_dependency 'fuzzy_match',   '>= 2.1.0', '< 2.2'
   spec.add_dependency 'amatch',        '>= 0.4.0', '< 0.5'
   spec.add_dependency 'jaro_winkler',  '>= 1.5.4', '< 1.6'
+  spec.add_dependency 'roo',           '>= 2.8.3', '< 2.9'
+  spec.add_dependency 'roo-xls',       '>= 1.2.0', '< 1.3'
 end

data/lib/eco/api/common/base_loader.rb

@@ -51,15 +51,19 @@ module Eco
         private
 
         def session
-          @session ||= ASSETS.session
+          ASSETS.session
         end
 
-        def micro
-          session.micro
+        def config
+          session.config
         end
 
-        def config
-          @config ||= ASSETS.config
+        def logger
+          session.logger
+        end
+
+        def micro
+          session.micro
         end
 
       end

data/lib/eco/api/common/loaders/parser.rb

@@ -19,6 +19,7 @@ module Eco
               @attribute = value
             end
 
+            # TODO: it migh rather merge?
             # Some parsers require dependencies to do their job.
             def dependencies(value = nil)
               @dependencies ||= {}

data/lib/eco/api/common/people/default_parsers.rb

@@ -44,3 +44,4 @@ require_relative 'default_parsers/freemium_parser'
 require_relative 'default_parsers/policy_groups_parser'
 require_relative 'default_parsers/login_providers_parser'
 require_relative 'default_parsers/csv_parser'
+require_relative 'default_parsers/xls_parser'

data/lib/eco/api/common/people/default_parsers/xls_parser.rb

@@ -0,0 +1,53 @@
+class Eco::API::Common::People::DefaultParsers::XLSParser < Eco::API::Common::Loaders::Parser
+  attribute :xls
+
+  attr_accessor :already_required
+  attr_reader   :file
+
+  def parser(file, deps)
+    @file = file
+    rows.tap {|r| @file = nil}
+  end
+
+  def serializer(array_hash, deps)
+    raise "Not implemented. TODO: using axlsx or rubyXL gems. See: https://spin.atomicobject.com/2017/03/22/parsing-excel-files-ruby/"
+  end
+
+  private
+
+  def headers
+    raise "You should implement this method"
+  end
+
+  def sheet_name
+    0
+  end
+
+  def workbook
+    require_reading_libs!
+    Roo::Spreadsheet.open(file)
+  end
+
+  def spreadheet(name_or_index = sheet_name)
+    workbook.sheet(name_or_index)
+  end
+
+  def rows(target = headers)
+    begin
+      spreadheet.parse(header_search: target)
+    rescue Roo::HeaderRowNotFoundError => e
+      missing  = JSON.parse(e.message)
+      logger.warn("The input file is missing these headers: #{missing}")
+      present = target - missing
+      rows(present)
+    end
+  end
+
+  def require_reading_libs!
+    return if already_required
+    require 'roo'
+    require 'roo-xls'
+    already_required = true
+  end
+
+end

data/lib/eco/api/common/people/entries.rb

@@ -99,6 +99,7 @@ module Eco
             newFrom to_a - discarded
           end
 
+          # TODO: it should rather use the the people-to-csv case somehow
           # Helper to dump the entries into a CSV
           # @param filename [String] the destination file
           def export(filename)

data/lib/eco/api/common/people/entry_factory.rb

@@ -2,6 +2,10 @@ module Eco
   module API
     module Common
       module People
+        # TODO: EntryFactory should suppport multiple schemas itself (rather that being done on `Session`)
+        # => currently, it's through session.entry_factory(schema: id), but this is wrong
+        # => This way, Entries and PersonEntry will be able to refer to attr_map and person_parser linked to schema_id
+        # => "schema_id" should be an optional column in the input file, or parsable via a custom parser to scope the schema
         # Helper factory class to generate entries (input entries).
         # @attr_reader schema [Ecoportal::API::V1::PersonSchema] person schema to be used in this entry factory
         class EntryFactory < Eco::API::Common::Session::BaseSession
@@ -12,7 +16,7 @@ module Eco
           # @param schema [Ecoportal::API::V1::PersonSchema] schema of person details that the parser will be based upon.
           # @param person_parser [nil, Eco::API::Common::People::PersonParser] set of attribute, type and format parsers/serializers.
           # @param attr_map [nil, Eco::Data::Mapper] attribute names mapper to translate external names into internal ones and _vice versa_.
-          def initialize(e, schema:, person_parser: nil, attr_map: nil)
+          def initialize(e, schema:, person_parser: nil, default_parser: nil, attr_map: nil)
             fatal "Constructor needs a PersonSchema. Given: #{schema}" if !schema.is_a?(Ecoportal::API::V1::PersonSchema)
             fatal "Expecting PersonParser. Given: #{person_parser}"    if person_parser && !person_parser.is_a?(Eco::API::Common::People::PersonParser)
             fatal "Expecting Mapper object. Given: #{fields_mapper}"   if attr_map && !attr_map.is_a?(Eco::Data::Mapper)
@@ -22,14 +26,25 @@ module Eco
             @source_person_parser = person_parser
 
             # load default parser + custom parsers
-            base_parser = Eco::API::Common::People::DefaultParsers.new(schema: @schema).merge(@source_person_parser)
+            @default_parser = default_parser&.new(schema: @schema) || Eco::API::Common::People::DefaultParsers.new(schema: @schema)
+            base_parser = @default_parser.merge(@source_person_parser)
             # new parser with linked schema
             @person_parser = @source_person_parser.new(schema: @schema).merge(base_parser)
             @person_parser_patch_version = @source_person_parser.patch_version
-
             @attr_map = attr_map
           end
 
+          def newFactory(schema: nil)
+            self.class.new(
+              environment,
+              schema: schema,
+              person_parser: @source_person_parser,
+              default_parser: @default_parser,
+              attr_map: @attr_map
+            )
+          end
+
+
           # provides with a Eco::API::Common::People::PersonParser object (collection of attribute parsers)
           # @note if the custom person parser has changed, it updates the copy of this EntryFactory instance
           # @return [Eco::API::Common::People::PersonParser] set of attribute, type and format parsers/serializers.
@@ -43,7 +58,7 @@ module Eco
 
           # key method to generate objects of `PersonEntry` that share dependencies via this `EntryFactory` environment.
           # @note this method is necessary to make the factory object work as a if it was a class `PersonEntry` you can call `new` on.
-          # @param data [Array<Hash>] data to be parsed. The external hashed entry.
+          # @param data [Hash, Person] data to be parsed/serialized. Parsed: the external hashed entry. Serialized: a Person object.
           # @return [Eco::API::Common::People::PersonEntry]
           def new(data, dependencies: {})
             PersonEntry.new(
@@ -73,26 +88,63 @@ module Eco
             fatal("Format should be a Symbol. Given '#{format}'") if format && !format.is_a?(Symbol)
             fatal("There is no parser/serializer for format ':#{format.to_s}'") unless no_format || @person_parser.defined?(format)
 
-            if file
-              arr_hash = []
-              if Eco::API::Common::Session::FileManager.file_exists?(file)
-                encoding   ||= Eco::API::Common::Session::FileManager.encoding(file)
-                encoding     = (encoding != "utf-8")? "#{encoding}|utf-8": encoding
-                file_content = File.read(file, encoding: encoding)
-                arr_hash     = person_parser.parse(format, file_content).map.each_with_index do |entry_hash, i|
-                  j = (format == :csv)? i + 2 : i + 1
-                  entry_hash.tap {|hash| hash["idx"] = j}
-                end
+            kargs = {}
+            kargs.merge!(content:  data)     unless no_data
+            kargs.merge!(file:     file)     unless no_file
+            kargs.merge!(format:   format)   unless no_format
+            kargs.merge!(encoding: encoding) if encoding
+
+            Entries.new(to_array_of_hashes(**kargs), klass: PersonEntry, factory: self)
+          end
+
+          def to_array_of_hashes(**kargs)
+            data = []
+            content, file, encoding, format = kargs.values_at(:content, :file, :encoding, :format)
+
+            # Support for multiple file
+            if file.is_a?(Array)
+              return file.each_with_object([]) do |f, out|
+                logger.info("Parsing file '#{f}'")
+                curr = to_array_of_hashes(**kargs.merge(file: f))
+                out.concat(curr)
+              end
+            end
+            # Get content only when it's not :xls
+            # note: even if content was provided, file takes precedence
+            content = get_file_content(file, format, encoding) if (format != :xls) && file
+
+            case content
+            when Hash
+              logger.error("Input data as 'Hash' not supported. Expecting 'Enumerable' or 'String'")
+              exit(1)
+            when String
+              to_array_of_hashes(content: person_parser.parse(format, content))
+            when Enumerable
+              sample = content.to_a.first
+              case sample
+              when Hash, Array, ::CSV::Row
+                Eco::CSV::Table.new(content).to_array_of_hashes
               else
-                logger.warn("File does not exist: #{file}")
+                logger.error("Input content 'Array' of '#{sample.class}' is not supported.")
               end
-
-              entries(data: arr_hash)
             else
-              Entries.new(data, klass: PersonEntry, factory: self)
+              if file && format == :xls
+                person_parser.parse(format, file)
+              else
+                logger.error("Could not obtain any data out of these: #{kargs}. Given content: '#{content.class}'")
+                exit(1)
+              end
+            end.tap do |out_array|
+              start_from_two = (format == :csv) || format == :xls
+              out_array.each_with_index do |entry_hash, i|
+                entry_hash["idx"] = start_from_two ? i + 2 : i + 1
+                entry_hash["source_file"] = file
+              end
             end
+
           end
 
+
           # Helper that generates a file out of `data:`.
           # @raise Exception
           #   - if you try to provide `data:` in the wrong format.
@@ -104,7 +156,7 @@ module Eco
           # @param format [Symbol] it specifies the format of the output `file:` (i.e. `:xml`, `:csv`). There must be a parser/serializer defined for it.
           # @param encoding [String] optional parameter to geneate `file:` content by unsing certain encoding.
           # @return [Void].
-          def export(data:, file: "export", format: :csv, encoding: "utf-8")
+          def export(data:, file: "export", format: :csv, encoding: "utf-8", internal_names: false)
             fatal("data: Expected Eco::API::Organization::People object. Given: #{data.class}") unless data.is_a?(Eco::API::Organization::People)
             fatal("A file should be specified.") unless !file.to_s.strip.empty?
             fatal("Format should be a Symbol. Given '#{format}'") if format && !format.is_a?(Symbol)
@@ -112,16 +164,18 @@ module Eco
 
             run = true
             if Eco::API::Common::Session::FileManager.file_exists?(file)
-              print "The file '#{file}' already exists. Do you want to overwrite it? (Y/n): "
-              res = STDIN.gets.strip
-              run = ["y", "Y", ""].include?(res)
+              prompt_user("The file '#{file}' already exists. Do you want to overwrite it? (Y/n):", default: "Y") do |response|
+                run = (response == "") || reponse.upcase.start_with?("Y")
+              end
             end
 
             if run
               deps = {"supervisor_id" => {people: data}}
 
               data_entries = data.map do |person|
-                self.new(person, dependencies: deps).external_entry
+                self.new(person, dependencies: deps).yield_self do |entry|
+                  internal_names ? entry.mapped_entry : entry.external_entry
+                end
               end
 
               File.open(file, "w", enconding: encoding) do |fd|
@@ -133,6 +187,17 @@ module Eco
 
           private
 
+          def get_file_content(file, format, encoding)
+            unless Eco::API::Common::Session::FileManager.file_exists?(file)
+              logger.error("File does not exist: #{file}")
+              exit(1)
+            end
+            ext        = File.extname(file)
+            encoding ||= Eco::API::Common::Session::FileManager.encoding(file)
+            encoding   = (encoding != "utf-8")? "#{encoding}|utf-8": encoding
+            content    = File.read(file, encoding: encoding)
+          end
+
           def fatal(msg)
             logger.fatal(msg)
             raise msg

data/lib/eco/api/common/people/person_entry.rb

@@ -216,6 +216,7 @@ module Eco
             end
           end
 
+          # TO DO: use person.details.schema_id to switch @emap and @person_parser (or just crash if they don't match?)
           # Setter to fill in all the schema `details` fields of the `Person` that are present in the `Entry`.
           # @note it only sets those details properties defined in the entry.
           #  Meaning that if an details property is not present in the entry, this will not be set on the target person.

data/lib/eco/api/common/people/person_parser.rb

@@ -16,7 +16,7 @@ module Eco
           CORE_ATTRS    = ["id", "external_id", "email", "name", "supervisor_id", "filter_tags", "freemium"]
           ACCOUNT_ATTRS = ["policy_group_ids", "default_tag", "send_invites", "landing_page_id", "login_provider_ids"]
           TYPE          = [:select, :text, :date, :number, :phone_number, :boolean, :multiple]
-          FORMAT        = [:csv, :xml, :json]
+          FORMAT        = [:csv, :xml, :json, :xls]
 
           attr_reader :schema
           attr_reader :details_attrs, :all_model_attrs

data/lib/eco/api/common/session.rb

@@ -13,4 +13,5 @@ require_relative 'session/sftp'
 require_relative 'session/s3_uploader'
 require_relative 'session/file_manager'
 require_relative 'session/environment'
+require_relative 'session/helpers'
 require_relative 'session/base_session'

data/lib/eco/api/common/session/base_session.rb

@@ -11,6 +11,8 @@ module Eco
           attr_reader  :api, :file_manager, :logger
           alias_method :fm, :file_manager
 
+          include Session::Helpers
+          
           def initialize(e)
             raise "Expected object Eco::API::Common::Session::Environment. Given: #{e.class}" unless e.is_a?(Environment)
             self.environment = e