eco-helpers 3.0.18 → 3.0.20

data/lib/eco/api/common/people/default_parsers/helpers/expected_headers.rb

@@ -0,0 +1,206 @@
+module Eco::API::Common::People
+  class DefaultParsers
+    module Helpers
+      module ExpectedHeaders
+        include Eco::Language::AuxiliarLogger
+
+        private
+
+        def require_headers!(raw_headers)
+          abort("Missing headers in CSV") unless raw_headers&.any?
+
+          empty = []
+          raw_headers.each_with_index do |header, idx|
+            empty << idx if header.to_s.strip.empty?
+          end
+
+          abort("Empty headers in column(s): #{empty.join(', ')}") if empty.any?
+
+          true
+        end
+
+        def check_headers!(raw_headers, order_check: false) # rubocop:disable Metrics/AbcSize
+          unmatch  = []
+          unmatch  = unmatched_headers(raw_headers) if order_check
+          missing  = missing_headers(raw_headers)
+          unknown  = unknown_headers(raw_headers)
+
+          criteria = [unknown, missing[:direct], missing[:indirect], unmatch]
+          return if criteria.all?(&:empty?)
+
+          msg = "Detected possible HEADER / FIELD  ISSUES !!!\n"
+
+          # requires exact match
+          unless unmatch.empty?
+            msg << "File headers/fields do NOT exactly match the expected:\n"
+            msg << "  * Expected: #{expected_headers}\n"
+
+            expected, given = unmatch.first
+            msg << "  * First unmatch => Given: '#{given}' where expected '#{expected}'\n"
+
+            missed = expected_headers - raw_headers
+
+            unless missed.empty?
+              msg << "  * Missing headers/fields:\n"
+              msg << "    - #{missed.join("\n    - ")}\n"
+            end
+          end
+
+          msg << "Missing or Wrong HEADER names in the file:\n"
+          msg << "  * UNKNOWN (or not used?): #{unknown}\n"  unless unknown.empty?
+          msg << "  * MISSING HEADER/FIELD: #{missing[:direct]}\n" unless missing[:direct].empty?
+
+          unless (data = missing[:indirect]).empty?
+            msg << "  * MISSING INDIRECTLY:\n"
+
+            data.each do |ext, info|
+              msg <<  "    - '#{ext}' => "
+              msg << (info[:attrs] || {}).map do |status, attrs|
+                if status == :inactive
+                  "makes inactive: #{attrs}"
+                elsif status == :active
+                  "there could be missing info in: #{attrs}"
+                end
+              end.compact.join("; ")
+
+              msg << "\n"
+            end
+          end
+
+          log(:warn) { msg }
+
+          msg = "There were issues identified on the file header/field names. Aborting..."
+          abort(msg) if options.dig(:input, :header_check, :must_be_valid)
+
+          sleep(2)
+        end
+
+
+        # @return [Hash] with missing `:direct` and `:indirect` attrs, where
+        #   - `:direct`   [Array] refers to direct attrs
+        #   - `:indirect` [Hash]  refers to indirect attrs that are `:active` or `:inactive`.
+        def missing_headers(raw_headers) # rubocop:disable Metrics/AbcSize
+          int_head = internal_present_or_active(raw_headers)
+
+          external = raw_headers.select do |e|
+            i = fields_mapper.to_internal(e)
+            int_head.include?(i)
+          end
+
+          ext_present = present_internal_expected_headers(int_head) | external
+          ext_miss    = expected_headers - ext_present
+
+          {
+            direct:   [],
+            indirect: {}
+          }.tap do |missing|
+            ext_miss.each do |ext|
+              next unless (int = fields_mapper.to_internal(ext))
+
+              missing[:direct] << ext if all_internal_attrs.include?(int)
+
+              related_attrs_requirements = required_attrs.values.select do |req|
+                dep       = req.dependant?(int)
+                affects   = dep && !int_head.include?(int)
+                in_header = int_head.include?(req.attr)
+                affects || (dep && !in_header)
+              end
+
+              next if related_attrs_requirements.empty?
+
+              data = missing[:indirect][ext] = {}
+              data[:int]   = int
+              data[:attrs] = {}
+
+              related_attrs_requirements.each_with_object(data[:attrs]) do |req, attrs|
+                status = req.active?(*int_head) ? :active : :inactive
+                attrs[status] ||= []
+                attrs[status] << req.attr
+              end
+            end
+          end
+        end
+
+
+        # The input file header names as expected
+        def expected_headers
+          @expected_headers ||= fields_mapper.list(:external).compact.uniq
+        end
+
+        def present_internal_expected_headers(internal_headers)
+          expected_headers.select do |ext|
+            int = fields_mapper.to_internal(ext)
+            internal_headers.include?(int)
+          end
+        end
+
+        def unmatched_headers(raw_headers)
+          expected_headers.zip(raw_headers).reject do |(expected, given)|
+            expected == given
+          end
+        end
+
+        def unknown_headers(raw_headers)
+          (raw_headers - expected_headers) - all_internal_attrs
+        end
+
+        def fields_mapper
+          session.fields_mapper
+        end
+
+        def required_attrs
+          @required_attrs ||= person_parser.required_attrs.to_h {|ra| [ra.attr, ra]}
+        end
+
+        def all_internal_attrs
+          @all_internal_attrs ||= [].tap do |int_attrs|
+            known_int_attrs  = person_parser.all_attrs(include_defined_parsers: true)
+            known_int_attrs |= fields_mapper.list(:internal).compact
+            int_attrs.concat(known_int_attrs)
+          end
+        end
+
+        # Scopes what internal attrs appear in headers as they are
+        def internal_present_or_active(raw_headers, inactive_requirements = {}) # rubocop:disable Metrics/AbcSize
+          # internal attrs that are not being mapped
+          int_all     = all_internal_attrs.reject {|i| fields_mapper.external?(i)}
+          hint        = raw_headers & int_all
+          hext        = raw_headers - hint
+          int_present = hint + hext.map {|e| fields_mapper.to_internal(e)}.compact
+
+          update_inactive = proc do
+            inactive_requirements.dup.each do |attr, req|
+              next unless req.active?(*int_present)
+
+              inactive_requirements.delete(attr)
+              int_present << attr
+              update_inactive.call
+            end
+          end
+
+          required_attrs.each_value do |req|
+            next if int_present.include?(req)
+
+            if req.active?(*int_present)
+              inactive_requirements.delete(req.attr)
+              int_present << req.attr
+              update_inactive.call
+            else
+              inactive_requirements[req.attr] = req
+            end
+          end
+
+          int_present
+        end
+
+        def person_parser
+          session.entry_factory.person_parser
+        end
+
+        def abort(msg)
+          super(msg, raising: false) if defined?(super)
+        end
+      end
+    end
+  end
+end

data/lib/eco/api/common/people/default_parsers/helpers/null_parsing.rb

@@ -0,0 +1,36 @@
+module Eco::API::Common::People
+  class DefaultParsers
+    module Helpers
+      module NullParsing
+        private
+
+        def parse_null(value)
+          return                           if null?(value)
+          return parse_null_on_hash(value) if value.is_a?(Hash)
+          return value                     unless value.is_a?(Array)
+
+          value.map {|val| parse_null(val)}
+        end
+
+        def parse_null_on_hash(value)
+          return value unless value.is_a?(Hash)
+
+          value.dup do |out|
+            value.each do |key, val|
+              next out.delete(key) unless (out[key] = parse_null(val))
+            end
+          end
+        end
+
+        def null?(value)
+          return true  if value.nil?
+          return false unless value.is_a?(String)
+          return true  if value.strip.to_s.empty?
+
+          str = value.strip.upcase
+          ['NULL'].any? {|token| str == token}
+        end
+      end
+    end
+  end
+end

data/lib/eco/api/common/people/default_parsers/helpers.rb

@@ -0,0 +1,15 @@
+require_relative 'helpers/expected_headers'
+require_relative 'helpers/null_parsing'
+
+module Eco
+  module API
+    module Common
+      module People
+        class DefaultParsers
+          module Helpers
+          end
+        end
+      end
+    end
+  end
+end

data/lib/eco/api/common/people/default_parsers/json_parser.rb

@@ -0,0 +1,56 @@
+class Eco::API::Common::People::DefaultParsers::JsonParser < Eco::API::Common::Loaders::Parser
+  attribute :json
+
+  include Eco::API::Common::People::DefaultParsers::Helpers::ExpectedHeaders
+  include Eco::API::Common::People::DefaultParsers::Helpers::NullParsing
+
+  def parser(filename, deps)
+    parse_json_file(filename).tap do |data|
+      # Don't enable header checks for now
+      next
+
+      puts 'Identifying unified json object keys...'
+      raw_headers = data.each_with_object([]) do |item, head|
+        head.concat(item.keys - head)
+      end
+
+      require_headers!(raw_headers)
+
+      next unless deps[:check_headers]
+      next unless check_headers?
+
+      check_headers!(
+        data,
+        order_check: false
+      )
+    end.each_with_object([]) do |item, arr_hash|
+      item_hash = item.keys.each_with_object({}) do |attr, hash|
+        next if attr.to_s.strip.empty?
+
+        hash[attr.strip] = parse_null(item[attr])
+      end
+
+      arr_hash.push(item_hash)
+    end
+  end
+
+  def serializer(array_hash, _deps)
+    array_hash.to_json
+  end
+
+  private
+
+  def check_headers?
+    !options.dig(:input, :header_check, :skip)
+  end
+
+  def parse_json_file(filename)
+    fd = File.open(filename)
+    JSON.load fd # rubocop:disable Security/JSONLoad
+  rescue JSON::ParserError => err
+    log(:error) { "Parsing error on file '#{filename}'" }
+    raise err
+  ensure
+    fd&.close
+  end
+end

data/lib/eco/api/common/people/default_parsers/xls_parser.rb

@@ -2,12 +2,9 @@ class Eco::API::Common::People::DefaultParsers::XLSParser < Eco::API::Common::Lo
   attribute :xls
 
   attr_accessor :already_required
-  attr_reader   :file
 
-  def parser(file, _deps)
-    @file = file
-    rows.tap do |rws|
-      @file = nil
+  def parser(filename, _deps)
+    rows(file: filename).tap do |rws|
       rws.each do |row|
         to_string!(row)
       end
@@ -22,13 +19,14 @@ class Eco::API::Common::People::DefaultParsers::XLSParser < Eco::API::Common::Lo
 
   def to_string!(row)
     row.transform_values! do |val|
-      next nil unless val
+      next     unless val
       next val if val.is_a?(String)
+
       val.to_s
     end
   end
 
-  def headers
+  def expected_headers
     log(:warn) {
       "Headers detection is using your fields_map.json file (native behaviour)"
     }
@@ -39,30 +37,31 @@ class Eco::API::Common::People::DefaultParsers::XLSParser < Eco::API::Common::Lo
     0
   end
 
-  def workbook
+  def workbook(file)
     require_reading_libs!
     Roo::Spreadsheet.open(file)
   end
 
-  def spreadheet(name_or_index = sheet_name)
-    workbook.sheet(name_or_index)
+  def spreadheet(name_or_index = sheet_name, file:)
+    workbook(file).sheet(name_or_index)
   end
 
-  def rows(target = headers)
-    spreadheet.parse(header_search: target, clean: true)
+  def rows(target = expected_headers, file:)
+    spreadheet(file: file).parse(header_search: target, clean: true)
   rescue Roo::HeaderRowNotFoundError => e
     missing = JSON.parse(e.message)
 
     log(:warn) {
-      "The input file is missing these headers: #{missing}"
+      "The input file is missing these expected headers: #{missing}"
     }
 
     present = target - missing
-    rows(present)
+    rows(present, file: file)
   end
 
   def require_reading_libs!
     return if already_required
+
     require 'roo'
     require 'roo-xls'
     self.already_required = true

data/lib/eco/api/common/people/default_parsers.rb

@@ -12,6 +12,7 @@ module Eco
   end
 end
 
+require_relative 'default_parsers/helpers'
 require_relative 'default_parsers/select_parser'
 require_relative 'default_parsers/boolean_parser'
 require_relative 'default_parsers/numeric_parser'
@@ -22,4 +23,5 @@ require_relative 'default_parsers/freemium_parser'
 require_relative 'default_parsers/policy_groups_parser'
 require_relative 'default_parsers/login_providers_parser'
 require_relative 'default_parsers/csv_parser'
+require_relative 'default_parsers/json_parser'
 require_relative 'default_parsers/xls_parser'

data/lib/eco/api/common/people/entry_factory.rb

@@ -28,6 +28,7 @@ module Eco
           #   to translate external names into internal ones and _vice versa_.
           def initialize(e, schema:, person_parser: nil, default_parser: nil, attr_map: nil)
             super(e)
+
             msg = "Constructor needs a PersonSchema. Given: #{schema.class}"
             fatal msg unless schema.is_a?(Ecoportal::API::V1::PersonSchema)
 
@@ -133,9 +134,10 @@ module Eco
                 out.concat(curr)
               end
             end
-            # Get content only when it's not :xls
+
+            # Get content only when it's not :xls, nor :json
             # note: even if content was provided, file takes precedence
-            if (format != :xls) && file # rubocop:disable Style/IfUnlessModifier
+            if get_content?(format) && file # rubocop:disable Style/IfUnlessModifier
               content = get_file_content(file, encoding: encoding)
             end
 
@@ -166,8 +168,10 @@ module Eco
               end
             end.tap do |out_array|
               start_from_two = (format == :csv) || format == :xls
-              out_array.each_with_index do |entry_hash, i|
-                entry_hash["idx"] = start_from_two ? i + 2 : i + 1
+              first_idx      = start_from_two ? 2 : 1
+
+              out_array.each.with_index(first_idx) do |entry_hash, idx|
+                entry_hash["idx"]         = idx
                 entry_hash["source_file"] = file
               end
             end
@@ -222,6 +226,13 @@ module Eco
 
           private
 
+          def get_content?(format)
+            return false if format == :xls
+            return false if format == :json
+
+            true
+          end
+
           def abort(message)
             log(:error) { message }
             exit(1)

data/lib/eco/api/common/session/sftp.rb

@@ -8,6 +8,7 @@ module Eco
           def initialize(enviro:)
             invalid_env = enviro && !enviro.is_a?(Eco::API::Common::Session::Environment)
             raise "Required Environment object (enviro:). Given: #{enviro}" if invalid_env
+
             @enviro = enviro
           end
 
@@ -103,6 +104,7 @@ module Eco
           def download(files, local_folder: nil, &block)
             puts "Creating local files:"
             created_files = []
+
             [files].flatten.compact.map do |fullname|
               basename = windows_basename(fullname)
               dest_fullname = File.join(local_folder || ".", basename)
@@ -110,6 +112,7 @@ module Eco
               created_files << dest_fullname
               sftp_session.download(fullname, dest_fullname)
             end.each(&:wait)
+
             # run SSH event loop while dw.active?
             created_files.tap { created_files.each(&block) }
           end
@@ -155,9 +158,11 @@ module Eco
           def windows_basename(remote_fullname)
             dir_sep = /[\\\/]/
             patr_re = /[<>:\\\/|?*]/
+
             parts   = remote_fullname.split(dir_sep).map do |node|
               node.gsub(patr_re, '_')
             end
+
             local_fullname = File.join(*parts)
             File.basename(local_fullname)
           end

data/lib/eco/api/custom/mailer.rb

@@ -5,5 +5,6 @@
 #  end
 class Eco::API::Custom::Mailer < Eco::API::Common::Loaders::Workflow::Mailer
   extend Eco::Language::Klass::WhenInherited
+
   when_inherited(&config_block)
 end

data/lib/eco/api/error.rb

@@ -96,9 +96,11 @@ module Eco
           end.sort do |k_1, k_2|
             next -1 if k_2 < k_1
             next  1 if k_1 < k_2
+
             0
           end.tap do |siblings|
             siblings.delete(Unclassified)
+
             if direct
               siblings.reject! do |si|
                 siblings.any? {|s| si < s}
@@ -120,8 +122,10 @@ module Eco
 
           descendants(direct: true).reverse.each do |klass|
             next unless klass.err_match?(err_msg)
+
             type = klass
             next unless klass.descendants?(direct: true)
+
             type = klass.get_type(err_msg, first: false) || type
           end
 

data/lib/eco/api/session/batch/job.rb

@@ -217,6 +217,7 @@ module Eco
             elsif !pqueue.empty?
               req_backup = as_update(pqueue, add_feedback: false)
               backup_update(req_backup)
+
               log(:debug) {
                 "Job ('#{name}':#{type}): going to launch batch against #{pqueue.count} entries"
               }
@@ -246,28 +247,25 @@ module Eco
           #   3. error messages in case they were errors from the server
           # @note if `launch` was not invoked, it specifies so
           # @return [String] the summary
-          def summary # rubocop:disable Metrics/AbcSize
+          def summary
             [].tap do |msg|
-              if pending?
-                msg << "PENDING - Batch #{type.to_s.upcase} - job '#{name}' - length: #{@queue.length}"
-              else
-                msg << feedback.generate(requests, only_stats: true)
+              msg << "PENDING Job -------->\n" if pending?
+              msg << feedback.generate(requests, only_stats: true)
 
-                if batch_policy && !batch_policy.compliant?(request_stats)
-                  msg << "Batch Policy Uncompliance:"
-                  msg << batch_policy.uncompliance(request_stats)
-                end
-
-                msg << status.errors.message if status
-                msg << subjobs_summary
+              if batch_policy && !batch_policy.compliant?(request_stats)
+                msg << 'Batch Policy Uncompliance:'
+                msg << batch_policy.uncompliance(request_stats)
               end
+
+              msg << status.errors.message if status
+              msg << subjobs_summary
             end.join("\n")
           end
 
           private
 
           def subjobs_summary
-            return "" unless subjobs.count.positive?
+            return '' unless subjobs.count.positive?
 
             [].tap do |msg|
               subjobs.map {|subjob| msg << subjob.summary}
@@ -431,7 +429,7 @@ module Eco
               next unless status.success?(entry)
 
               if type == :create && entry.respond_to?(:id=)
-                entry.id = status[entry].body["id"].tap do |id|
+                entry.id = status[entry].body['id'].tap do |id|
                   next unless id.to_s.strip.empty?
 
                   ref = Eco::API::Session::Batch::Feedback.person_ref(entry)
@@ -503,9 +501,9 @@ module Eco
 
           # Keep a copy of the requests for future reference
           def backup_update(requests, simulate: false)
-            dry_run   = simulate ? "_dry_run" : ""
+            dry_run   = simulate ? '_dry_run' : ''
             dir       = config.people.requests_folder
-            filename  = name.split(" ").join("-").gsub(/[=\\\/><,"-]+/, "_") # rubocop:disable Style/RedundantArgument
+            filename  = name.split(' ').join('-').gsub(/[=\\\/><,"-]+/, "_") # rubocop:disable Style/RedundantArgument
             file      = File.join(dir, "#{type}_data_#{filename}#{dry_run}.json")
             file_manager.save_json(requests, file, :timestamp)
           end

data/lib/eco/api/session/batch/jobs.rb

@@ -81,7 +81,7 @@ module Eco
           end
 
           def pending?
-            any? {|job| job.pending?}
+            any?(&:pending?)
           end
 
           # Launches every `Batch::Job` in the group.
@@ -108,19 +108,17 @@ module Eco
             end
           end
 
-          def status
+          def status(&block)
             if block_given?
-              status.each do |job, job_status|
-                yield(job, job_status)
-              end
+              status.each(&block)
               self
-            else # rubocop:disable Naming/MemoizedInstanceVariableName
-              @jobs_status ||= {}
+            else
+              @jobs_status ||= {} # rubocop:disable Naming/MemoizedInstanceVariableName
             end
           end
 
           def errors?
-            any? {|job| job.errors?}
+            any?(&:errors?)
           end
 
           def summary

data/lib/eco/api/session/batch/launcher/mode_size.rb

@@ -18,14 +18,17 @@ module Eco
             private
 
             # Swaps to batch endpoint on specific errors
-            def batch_mode_on(*error_types, options: self.options, allow_job_mode: true, &block)
+            def batch_mode_on(*error_types, options: self.options, allow_job_mode: true)
+              msg = "Expecting block. Non given"
+              raise ArgumentError, msg unless block_given?
+
               in_job_mode = allow_job_mode && job_mode?(options)
 
               yield(in_job_mode, batch_size(options))
             rescue *error_types
               raise unless in_job_mode
 
-              yield(false , batch_mode_size)
+              yield(false, batch_mode_size)
             end
 
             # MODE

data/lib/eco/api/session/batch/launcher/retry.rb

@@ -25,7 +25,12 @@ module Eco
               explanation << "You have #{retries_left} retries left."
               question    = " Do you want to retry (y/N)?"
 
-              prompt_user(question, default: "Y", explanation: explanation, timeout: 10) do |response|
+              prompt_user(
+                question,
+                default:     "Y",
+                explanation: explanation,
+                timeout:     10
+              ) do |response|
                 raise unless response.upcase.start_with?("Y")
 
                 puts "\nOkay... let's retry!"

data/lib/eco/api/session/batch/launcher/status_handling.rb

@@ -6,14 +6,16 @@ module Eco
           module StatusHandling
             private
 
-            def tap_status(enviro:, queue:, method:, status: nil, &block)
+            def tap_status(enviro:, queue:, method:, status: nil)
               status ||= Eco::API::Session::Batch::Status.new(
                 enviro,
                 queue:  queue,
                 method: method
               )
 
-              status.tap(&block)
+              status.tap do
+                yield(status) if block_given?
+              end
             end
           end
         end