eco-helpers 3.0.16 → 3.0.18

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: baeb42abbb1d31497328afd4c813dec4acfebb598546ad55a4f1265a45dc89b5
-  data.tar.gz: 7d4c88626946b84e1ea22f3b27eb7570a9daa6d2962036e6fbf667b000dd5996
+  metadata.gz: 47c28bbf13f50c0f3f895c40c0ad8532d13233e0e2b81faea474bbeb0a1c2308
+  data.tar.gz: e13091192acd044b667cd20ebe7bcaf5c17da30c1ef83266fefa9ffa90fb8348
 SHA512:
-  metadata.gz: 8af8fdbccacb57655756a01f0c30ff1d38c58e232f77fedc5aeb058cb7260ed1863834af1995d7268d355e09c00fedc671b4417f1f14d76c25c9609a038b47d3
-  data.tar.gz: c4daf6291086d8b15f5eb1606291e8161e0e0326b4f5ed8e917e95d09b9c1de7ff2665e44ec3ed35986d0cd07ec232dfe40822d0d32bbd12020f7290a9ce1405
+  metadata.gz: 02fe482f0c75e0926fe147e1ab3f4b0a0d1b826990bae3ad8ea97fb0aa53a800e1d208fb62e209877c0594b4f32792096967ec1f0528be8c81c3cd8c404c5131
+  data.tar.gz: 4f9c12554b25c81243eac348f106392a092853c87204922d25cd1f25d287abcc23461df2b0a6a19b3dd3e4624f625e49ce5920ed04000ab6641f26312a38cb0f

data/CHANGELOG.md

@@ -2,18 +2,34 @@
 
 All notable changes to this project will be documented in this file.
 
-## [3.0.17] - 2024-10-xx
+## [3.0.18] - 2024-10-xx
 
 ### Added
 
 ### Changed
 
+- upgrade `ecoportal-api` gem
+- upgrade `ecoportal-api-v2` gem
+- `Eco::API::Session::Batch#launch_batch`
+  - **removed** `per_page` parameter
+  - **added** auto-swap endpoint from `job` to `batch` on TimeOut errors.
+  - **retries** also on `Ecoportal::API::Errors::StartTimeOut`
+
 ### Fixed
 
-## [3.0.16] - 2024-10-15
+## [3.0.17] - 2024-10-18
 
 ### Added
 
+- Configurability around the **size** of batch and job modes
+
+### Changed
+
+- `ecoportal-api` gem upgrade
+- Internal refactor of `Eco::API::Session::Batch`
+
+## [3.0.16] - 2024-10-15
+
 ### Changed
 
 - `Eco::API::Session::Batch#launch_batch`

data/eco-helpers.gemspec

@@ -40,9 +40,9 @@ Gem::Specification.new do |spec|
   spec.add_dependency 'bcrypt_pbkdf',  '~> 1.0'
   spec.add_dependency 'docx',          '>= 0.8.0', '< 0.9'
   spec.add_dependency 'dotenv',        '~> 3'
-  spec.add_dependency 'ecoportal-api',         '~> 0.10', '>= 0.10.4'
+  spec.add_dependency 'ecoportal-api',         '~> 0.10', '>= 0.10.6'
   spec.add_dependency 'ecoportal-api-graphql', '~> 0.4',  '>= 0.4.2'
-  spec.add_dependency 'ecoportal-api-v2',      '~> 2.0',  '>= 2.0.10'
+  spec.add_dependency 'ecoportal-api-v2',      '~> 2.0',  '>= 2.0.11'
   spec.add_dependency 'ed25519',       '~> 1.2'
   spec.add_dependency 'fast_excel',    '>= 0.5.0',  '< 0.6'
   spec.add_dependency 'fuzzy_match',   '>= 2.1.0',  '< 2.2'

data/lib/eco/api/session/batch/launcher/benchmarking.rb

@@ -0,0 +1,23 @@
+module Eco
+  module API
+    class Session
+      class Batch
+        module Launcher
+          module Benchmarking
+            private
+
+            def str_per_sec(start, count)
+              now  = Time.now
+              secs = (now - start).round(2)
+
+              return ' -- ' unless secs > 0.0
+
+              per_sec = (count.to_f / secs).round(2)
+              "#{secs}s -> #{per_sec} people/s"
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/eco/api/session/batch/launcher/mode_size.rb

@@ -0,0 +1,65 @@
+module Eco
+  module API
+    class Session
+      class Batch
+        module Launcher
+          module ModeSize
+            include Eco::API::Session::Batch::Launcher::Options
+
+            DEFAULT_BATCH_SIZE = 50
+            DEFAULT_JOB_SIZE   = 100
+
+            def batch_size(opts = options)
+              return job_mode_size if job_mode?(opts)
+
+              batch_mode_size
+            end
+
+            private
+
+            # Swaps to batch endpoint on specific errors
+            def batch_mode_on(*error_types, options: self.options, allow_job_mode: true, &block)
+              in_job_mode = allow_job_mode && job_mode?(options)
+
+              yield(in_job_mode, batch_size(options))
+            rescue *error_types
+              raise unless in_job_mode
+
+              yield(false , batch_mode_size)
+            end
+
+            # MODE
+
+            # @return [Symbol] the batch mode to run
+            def batch_mode(opts = options)
+              opts.dig(:workflow, :batch, :mode) || :batch
+            end
+
+            # @return [Boolean] are we running in `:job` mode?
+            def job_mode?(opts = options)
+              batch_mode(opts) == :job
+            end
+
+            # SIZE
+
+            def job_mode_size
+              options.dig(:workflow, :batch, :job, :size).then do |size|
+                next self.class::DEFAULT_JOB_SIZE unless size
+
+                size
+              end
+            end
+
+            def batch_mode_size
+              options.dig(:workflow, :batch, :size).then do |size|
+                next self.class::DEFAULT_BATCH_SIZE unless size
+
+                [size, 100].min
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/eco/api/session/batch/launcher/options.rb

@@ -0,0 +1,21 @@
+module Eco
+  module API
+    class Session
+      class Batch
+        module Launcher
+          module Options
+            private
+
+            # Default way to retrieve options (unless provided)
+            def options
+              return @options if instance_variable_defined?(:@options)
+              return super    if defined?(super)
+
+              ASSETS.cli.options
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/eco/api/session/batch/launcher/retry.rb

@@ -0,0 +1,40 @@
+module Eco
+  module API
+    class Session
+      class Batch
+        module Launcher
+          module Retry
+            def self.included(base)
+              unless base <= Eco::API::Session::Batch::Launcher
+                msg  = "To be included only in Eco::API::Common::Session::BaseSession. "
+                msg << "Tried on '#{base}'"
+                raise msg
+              end
+
+              super
+            end
+
+            private
+
+            def offer_retry_on(*error_types, retries_left: 3, &block)
+              yield
+            rescue *error_types => err
+              raise err.class, err.message, cause: nil unless retries_left.positive?
+
+              explanation = "#{err}\n"
+              explanation << "You have #{retries_left} retries left."
+              question    = " Do you want to retry (y/N)?"
+
+              prompt_user(question, default: "Y", explanation: explanation, timeout: 10) do |response|
+                raise unless response.upcase.start_with?("Y")
+
+                puts "\nOkay... let's retry!"
+                offer_retry_on(*error_types, retries_left: retries_left - 1, &block)
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/eco/api/session/batch/launcher/status_handling.rb

@@ -0,0 +1,23 @@
+module Eco
+  module API
+    class Session
+      class Batch
+        module Launcher
+          module StatusHandling
+            private
+
+            def tap_status(enviro:, queue:, method:, status: nil, &block)
+              status ||= Eco::API::Session::Batch::Status.new(
+                enviro,
+                queue:  queue,
+                method: method
+              )
+
+              status.tap(&block)
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/eco/api/session/batch/launcher/valid_methods.rb

@@ -0,0 +1,34 @@
+module Eco
+  module API
+    class Session
+      class Batch
+        module Launcher
+          module ValidMethods
+            VALID_METHODS = %i[get create update upsert delete].freeze
+
+            def self.included(base)
+              super
+              base.extend(ClassMethods)
+              base.send(:include, InstanceMethods)
+            end
+
+            module ClassMethods
+              # @return [Boolean] `true` if the method is supported, `false` otherwise.
+              def valid_method?(value)
+                VALID_METHODS.include?(value)
+              end
+            end
+
+            module InstanceMethods
+              private
+
+              def valid_method?(value)
+                self.class.valid_method?(value)
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/eco/api/session/batch/launcher.rb

@@ -0,0 +1,146 @@
+require_relative 'launcher/valid_methods'
+require_relative 'launcher/options'
+require_relative 'launcher/mode_size'
+require_relative 'launcher/benchmarking'
+require_relative 'launcher/status_handling'
+require_relative 'launcher/retry'
+
+module Eco
+  module API
+    class Session
+      class Batch
+        module Launcher
+          def self.included(base)
+            unless base <= Eco::API::Common::Session::BaseSession
+              msg  = "To be included only in Eco::API::Common::Session::BaseSession. "
+              msg << "Tried on '#{base}'"
+              raise msg
+            end
+
+            super
+
+            base.send(:include, ValidMethods)
+          end
+
+          include Options
+          include ModeSize
+          include Benchmarking
+          include StatusHandling
+          include Retry
+
+          TIMEOUT_RETRIES = 2
+          RETRY_ON        = [
+            Ecoportal::API::Errors::TimeOut,
+            Ecoportal::API::Errors::StartTimeOut
+          ].freeze
+
+          private
+
+          def batch_from(
+            data,
+            method:,
+            params:  {},
+            silent:  false,
+            options: self.options
+          )
+            fatal "Invalid batch method: #{method}." unless valid_method?(method)
+            return unless data.is_a?(Enumerable)
+
+            msg = "cannot batch #{method} without api connnection, please provide a valid api connection!"
+            fatal msg unless (people_api = api&.people)
+
+            launch_batch(
+              data,
+              method:     method,
+              people_api: people_api,
+              silent:     silent,
+              options:    options
+            )
+          end
+
+          def launch_batch( # rubocop:disable Metrics/AbcSize
+            data,
+            method:,
+            status:     nil,
+            job_mode:   true,
+            options:    self.options,
+            people_api: api&.people,
+            silent:     false
+          )
+
+            tap_status(status: status, enviro: enviro, queue: data, method: method) do |overall_status|
+              pending_for_server_error = data.to_a[0..]
+
+              batch_mode_on(*RETRY_ON, options: options, allow_job_mode: job_mode) do |job_mode, per_page|
+                iteration  = 0
+                done       = 0
+                iterations = (data.length.to_f / per_page).ceil
+
+                start_time = Time.now
+
+                data.each_slice(per_page) do |slice|
+                  iteration  += 1
+
+                  msg  = "starting batch '#{method}' iteration #{iteration}/#{iterations}, "
+                  msg << "with #{slice.length} entries of #{data.length} -- #{done} done"
+                  msg << (" " * 20)
+                  log(:info) { msg } unless silent
+
+                  start_slice = Time.now
+
+                  offer_retry_on(*RETRY_ON, retries_left: TIMEOUT_RETRIES) do
+                    people_api.batch(job_mode: job_mode) do |batch|
+                      slice.each do |person|
+                        batch.public_send(method, person) do |response|
+                          faltal("Request with no response") unless response
+
+                          next if server_error?(response)
+
+                          pending_for_server_error.delete(person)
+                          overall_status[person] = response
+                        end
+                      end
+                    end # end batch
+                  end
+
+                  done += slice.length
+
+                  msg  = "  ... iteration #{iteration}/#{iterations} done "
+                  msg << "in #{str_per_sec(start_slice, slice.length)} "
+                  msg << "(average: #{str_per_sec(start_time, done)})"
+                  msg << (" " * 20)
+                  log(:info) { msg } unless silent
+                end # next slice
+              end
+
+              # temporary working around (due to back-end problems with batch/jobs)
+              unless pending_for_server_error.empty?
+                msg = "Going to re-try #{pending_for_server_error.count} due to server errors"
+                log(:info) { msg } unless silent
+
+                launch_batch(
+                  pending_for_server_error,
+                  status:     overall_status,
+                  method:     method,
+                  job_mode:   false,
+                  per_page:   per_page,
+                  people_api: people_api,
+                  silent:     silent,
+                  options:    options
+                )
+              end
+            end
+          end
+
+          def server_error?(response)
+            res_status   = response.status
+            server_error = !res_status || res_status.server_error?
+            other_error  = !server_error && (!res_status.code || res_status.code < 100)
+            no_body      = !server_error && !other_error && !response.body
+            server_error || other_error || no_body
+          end
+        end
+      end
+    end
+  end
+end

data/lib/eco/api/session/batch/request_stats.rb

@@ -6,48 +6,48 @@ module Eco
         # @attr_reader stats [Hash] plain `Hash` with the number of requests that include an attribute
         class RequestStats
           CORE_ATTRS     = Eco::API::Common::People::PersonParser::CORE_ATTRS
-          ACCOUNT_ATTRS  = (Eco::API::Common::People::PersonParser::ACCOUNT_ATTRS + ["permissions_custom"]).uniq
-          DETAILS_ATTRS  = ["fields"]
-          BLANKED_PREFIX = "blanked_"
-          DETAILS_FIELDS = "details_fields"
+          ACCOUNT_ATTRS  = (Eco::API::Common::People::PersonParser::ACCOUNT_ATTRS + ['permissions_custom']).uniq
+          DETAILS_ATTRS  = ['fields'].freeze
+          BLANKED_PREFIX = 'blanked_'.freeze
+          DETAILS_FIELDS = 'details_fields'.freeze
 
           class << self
-
             def valid_type?(type)
               Eco::API::Session::Batch::Job.valid_type?(type.to_sym)
             end
 
             def core_attrs(stats: false, all: false)
               CORE_ATTRS.dup.tap do |attrs|
-                if stats || all
-                  attrs.unshift("core")
-                  attrs.concat(blank_attrs(CORE_ATTRS))
-                end
+                next unless stats || all
+
+                attrs.unshift('core')
+                attrs.concat(blank_attrs(CORE_ATTRS))
               end
             end
 
             def account_attrs(stats: false, all: false)
               ACCOUNT_ATTRS.dup.tap do |attrs|
-                if stats || all
-                  attrs.unshift("account_remove")
-                  attrs.unshift("account") if all
-                  attrs.concat(blank_attrs(ACCOUNT_ATTRS))
-                end
+                next unless stats || all
+
+                attrs.unshift('account_remove')
+                attrs.unshift('account') if all
+                attrs.concat(blank_attrs(ACCOUNT_ATTRS))
               end
             end
 
             def details_attrs(stats: false, all: false)
               DETAILS_ATTRS.dup.tap do |attrs|
-                if stats || all
-                  attrs.unshift("details_remove")
-                  attrs.unshift("details") if all
-                end
+                next unless stats || all
+
+                attrs.unshift('details_remove')
+                attrs.unshift('details') if all
               end
             end
 
             def blanked_prefix(attr = nil)
               @blanked_prefix ||= BLANKED_PREFIX
               return @blanked_prefix unless attr
+
               "#{blanked_prefix}#{attr}"
             end
 
@@ -58,15 +58,16 @@ module Eco
             def blank_attrs(attrs)
               attrs.map {|attr| "#{blanked_prefix}#{attr}"}
             end
-
           end
 
           attr_reader :type, :count
 
           def initialize(type:, requests: [])
-            raise "type should be one of #{Eco::API::Session::Batch::Job.types}. Given: #{type}"  unless self.class.valid_type?(type.to_sym)
+            msg = "Type should be one of #{Eco::API::Session::Batch::Job.types}. Given: #{type}"
+            raise msg unless self.class.valid_type?(type.to_sym)
+
             @type  = type.to_sym
-            @count = requests && requests.length
+            @count = requests&.length
             @stats = build(requests)
           end
 
@@ -86,21 +87,22 @@ module Eco
 
           def attr_value(attr, percent: false, total: count, details: false)
             target = details ? (@stats[DETAILS_FIELDS] || {}) : @stats
-            i = target["#{attr}"]
+            i      = target[attr.to_s]
             return i unless percent
+
             percentage(i, total: total)
           end
 
           def core(percent: false)
-            attr_value("core", percent: percent)
+            attr_value('core', percent: percent)
           end
 
           def account(percent: false)
-            attr_value("account", percent: percent)
+            attr_value('account', percent: percent)
           end
 
           def details(percent: false)
-            attr_value("details", percent: percent)
+            attr_value('details', percent: percent)
           end
 
           def stats
@@ -110,13 +112,20 @@ module Eco
           def build(requests)
             stats.tap do |stats|
               stats[type] = count
-              unless !requests || !requests.is_a?(Enumerable) || requests.empty?
-                stats[DETAILS_FIELDS] = Hash.new(0)
-                requests.each_with_index do |request|
-                  add_core_stats(stats, request || {})
-                  add_account_stats(stats, request || {})
-                  add_details_stats(stats, request || {})
-                end
+
+              no_requests =
+                !requests                   ||
+                !requests.is_a?(Enumerable) ||
+                requests.empty?
+
+              next if no_requests
+
+              stats[DETAILS_FIELDS] = Hash.new(0)
+
+              requests.each do |request|
+                add_core_stats(stats, request || {})
+                add_account_stats(stats, request || {})
+                add_details_stats(stats, request || {})
               end
             end
           end
@@ -124,74 +133,84 @@ module Eco
           def attrs_to_stat(stats, hash, attrs)
             stats.tap do |st|
               attrs.each do |attr|
-                if hash.key?(attr)
-                  st[attr] += 1
-                  st[blanked_prefix(attr)]+= 1 if blanked_value?(hash[attr])
-                end
+                next unless hash.key?(attr)
+
+                st[attr] += 1
+                next unless blanked_value?(hash[attr])
+
+                st[blanked_prefix(attr)]+= 1
               end
             end
           end
 
           def add_core_stats(stats, request)
-            stats["core"] += 1 if (request.keys & core_attrs).length > 0
+            any_core = !(request.keys & core_attrs).empty? # rubocop:disable Style/ArrayIntersect
+            stats['core'] += 1 if any_core
+
             attrs_to_stat(stats, request, core_attrs)
           end
 
           def add_account_stats(stats, request)
-            if request.key?("account")
-              stats["account"] += 1
-              stats["account_remove"] += 1 if !request["account"]
-              attrs_to_stat(stats, request["account"] || {}, account_attrs)
-            end
+            return unless request.key?('account')
+
+            stats['account'] += 1
+            stats['account_remove'] += 1 unless request['account']
+
+            attrs_to_stat(stats, request['account'] || {}, account_attrs)
           end
 
           def add_details_stats(stats, request)
-            if request.key?("details")
-              stats["details"] += 1
-              stats["details_remove"] += 1 if !request["details"]
-
-              det_attrs = {}
-              if fields = request.dig("details", "fields")
-                stats["fields"] += fields.length
-                det_attrs = fields.each_with_object(det_attrs) {|fld, hash| hash[fld["alt_id"]] = fld["value"]}
-              end
+            return unless request.key?('details')
+
+            stats['details'] += 1
+            stats['details_remove'] += 1 unless request['details']
+
+            det_attrs = {}
+            if (fields = request.dig('details', 'fields'))
+              stats['fields'] += fields.length
 
-              attrs_to_stat(stats[DETAILS_FIELDS], det_attrs, det_attrs.keys)
+              det_attrs = fields.each_with_object(det_attrs) do |fld, hash|
+                hash[fld['alt_id']] = fld['value']
+              end
             end
+
+            attrs_to_stat(stats[DETAILS_FIELDS], det_attrs, det_attrs.keys)
           end
 
           def core_pairs(percent: false)
-            cattrs        = core_attrs + blank_attrs(core_attrs)
-            [["core",     core(percent: percent)]] + pairs(cattrs, percent: percent, total: core)
+            cattrs    = core_attrs + blank_attrs(core_attrs)
+            [['core', core(percent: percent)]] + pairs(cattrs, percent: percent, total: core)
           end
 
           def account_pairs(percent: false)
-            aattrs         = ["account_remove"] + account_attrs + blank_attrs(account_attrs)
-            [["account",   account(percent: percent)]] + pairs(aattrs, percent: percent, total: account)
+            aattrs       = ['account_remove'] + account_attrs + blank_attrs(account_attrs)
+            [['account', account(percent: percent)]] + pairs(aattrs, percent: percent, total: account)
           end
 
           def details_pairs(percent: false)
-            det_pairs  = [["details", details(percent: percent)]]
-            det_pairs += [["fields", fields_average]] if attr_value("fields") && fields_average
-            det_pairs += pairs(["details_remove"], percent: percent, total: details)
+            det_pairs  = [['details', details(percent: percent)]]
+            det_pairs += [['fields', fields_average]] if attr_value('fields') && fields_average
+            det_pairs += pairs(['details_remove'], percent: percent, total: details)
             det_pairs += pairs(details_field_attrs, percent: percent, total: details, details: true)
+            det_pairs
           end
 
           def pairs(attrs, percent: false, total: count, details: false)
-            pairs = attrs.map do |a|
-              (v = attr_value(a, percent: percent, total: count, details: details)) > 0 ? [a, v] : nil
+            attrs.map do |a|
+              value = attr_value(a, percent: percent, total: total, details: details)
+              value.positive? ? [a, value] : nil
             end.compact
           end
 
           def pairs_to_line(pairs, percent: false)
-            key_val_delimiter = ": "; attr_delimiter = " ++ "
+            key_val_delimiter = ': '; attr_delimiter = ' ++ '
             pairs.map do |p|
-              [p.first.to_s, "#{p.last.to_s}" + (percent ? "%" : "")].join(key_val_delimiter)
+              [p.first.to_s, "#{p.last}#{percent ? '%' : ''}"].join(key_val_delimiter)
             end.join(attr_delimiter)
           end
 
           def core_attrs
-            @core_attrs    ||= self.class.core_attrs
+            @core_attrs ||= self.class.core_attrs
           end
 
           def account_attrs
@@ -216,12 +235,10 @@ module Eco
 
           def blanked_value?(value)
             case value
-            when nil
-              true
-            when false
+            when nil, false
               true
             when Numeric
-              value == 0
+              value.zero?
             when Array
               value.compact.empty?
             when String
@@ -230,16 +247,17 @@ module Eco
           end
 
           def fields_average
-            if (fields_num = attr_value("fields")) && (total = details) > 0
-              (fields_num.to_f / total.to_f).round(2)
-            end
+            fields_num = attr_value('fields')
+            return unless fields_num && (total = details).positive?
+
+            (fields_num / total.to_f).round(2)
           end
 
           def percentage(num, total: count)
+            return unless num
+
             total ||= count
-            if num
-              (num.to_f / total * 100).round(2)
-            end
+            (num.to_f / total * 100).round(2)
           end
         end
       end

data/lib/eco/api/session/batch/searcher.rb

@@ -0,0 +1,108 @@
+require_relative 'launcher/valid_methods'
+require_relative 'launcher/options'
+
+module Eco
+  module API
+    class Session
+      class Batch
+        module Searcher
+          def self.included(base)
+            unless base <= Eco::API::Session::Batch
+              msg  = "To be included only in Eco::API::Common::Session::BaseSession. "
+              msg << "Tried on '#{base}'"
+              raise msg
+            end
+
+            super
+          end
+
+          # Gets the _people_ of the organization according `params`.
+          #   If `people` is not `nil`, scopes to only the people specified.
+          # @note
+          #   - If `people` is given keys `page:` and `q` of `params:`.
+          # @param people [Nil, People, Enumerable<Person>, Enumerable<Hash>] target _People_ to launch the batch against.
+          # @param params [Hash] api request options.
+          # @option params [String] :page the page number `page` based on `:per_page`.
+          # @option params [String] :per_page the number of people included per each batch api request.
+          # @option params [String] :q some text to search. Omit this parameter to target all the people.
+          # @return [Array<People>] all the people based on `params`
+          def get_people(people = nil, params: {}, silent: false, options: self.options)
+            return get(params: params, silent: silent, options: options) unless people.is_a?(Enumerable)
+
+            launch(
+              people,
+              method:  :get,
+              params:  params,
+              silent:  silent,
+              options: options
+            ).people
+          end
+
+          def search(data, silent: false, params: {}, options: self.options) # rubocop:disable Metrics/AbcSize
+            params = {per_page: batch_size(options)}.merge(params)
+
+            launch(
+              data,
+              method:  :get,
+              params:  params,
+              silent:  silent,
+              options: options
+            ).tap do |status|
+              status.mode = :search
+
+              entries = status.queue
+              puts "\n"
+
+              entries.each_with_index do |entry, i|
+                if (i % 10).zero?
+                  percent = i * 100 / entries.length
+                  print "Searching: #{percent.round}% (#{i}/#{entries.length} entries)\r"
+                  $stdout.flush
+                end
+
+                next if status.success?(entry)
+
+                email = nil
+                if entry.respond_to?(:email)
+                  email = entry.email
+                elsif entry.respond_to?(:to_h)
+                  email = entry.to_h["email"]
+                end
+
+                people_matching = []
+                email = email.to_s.strip.downcase
+
+                unless email.empty?
+                  people_matching = get(
+                    params:  params.merge(q: email),
+                    silent:  silent,
+                    options: options
+                  ).select do |person|
+                    person.email == email
+                  end
+                end
+
+                case people_matching.length
+                when 1
+                  status.set_person_match(entry, people_matching.first)
+                when 2..Float::INFINITY
+                  status.set_people_match(entry, people_matching)
+                end
+              end
+            end
+          end
+
+          private
+
+          def get(params: {}, silent: false, options: self.options)
+            msg = "cannot batch get without api connnection, please provide a valid api connection!"
+            fatal msg unless (people_api = api&.people)
+
+            params = {per_page: batch_size(options)}.merge(params)
+            people_api.get_all(params: params, silent: silent)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/eco/api/session/batch.rb

@@ -2,54 +2,11 @@ module Eco
   module API
     class Session
       class Batch < Common::Session::BaseSession
-        DEFAULT_BATCH_SIZE = 50
-        DEFAULT_JOB_SIZE   = 100
-        VALID_METHODS      = %i[get create update upsert delete].freeze
+        require_relative 'batch/launcher'
+        require_relative 'batch/searcher'
 
-        class << self
-          # @return [Boolean] `true` if the method is supported, `false` otherwise.
-          def valid_method?(value)
-            VALID_METHODS.include?(value)
-          end
-        end
-
-        def batch_size(opts = options)
-          return self.class::DEFAULT_JOB_SIZE if job_mode?(opts)
-
-          self.class::DEFAULT_BATCH_SIZE
-        end
-
-        # @return [Symbol] the batch mode to run
-        def batch_mode(opts = options)
-          opts.dig(:workflow, :batch, :mode) || :batch
-        end
-
-        # @return [Boolean] are we running in `:job` mode?
-        def job_mode?(opts = options)
-          batch_mode(opts) == :job
-        end
-
-        # Gets the _people_ of the organization according `params`.
-        #   If `people` is not `nil`, scopes to only the people specified.
-        # @note
-        #   - If `people` is given keys `page:` and `q` of `params:`.
-        # @param people [Nil, People, Enumerable<Person>, Enumerable<Hash>] target _People_ to launch the batch against.
-        # @param params [Hash] api request options.
-        # @option params [String] :page the page number `page` based on `:per_page`.
-        # @option params [String] :per_page the number of people included per each batch api request.
-        # @option params [String] :q some text to search. Omit this parameter to target all the people.
-        # @return [Array<People>] all the people based on `params`
-        def get_people(people = nil, params: {}, silent: false, options: self.options)
-          return get(params: params, silent: silent, options: options) unless people.is_a?(Enumerable)
-
-          launch(
-            people,
-            method:  :get,
-            params:  params,
-            silent:  silent,
-            options: options
-          ).people
-        end
+        include Launcher
+        include Searcher
 
         # launches a batch of `method` type using `people` and the specified `params`
         # @raise Exception
@@ -60,222 +17,8 @@ module Eco
         # @param params [Hash] api request options.
         # @option params [String] :per_page the number of people included per each batch api request.
         # @return [Batch::Status] the `status` of this batch launch.
-        def launch(people, method:, params: {}, silent: false, options: self.options)
-          batch_from(
-            people,
-            method:  method,
-            params:  params,
-            silent:  silent,
-            options: options
-          )
-        end
-
-        def search(data, silent: false, params: {}, options: self.options) # rubocop:disable Metrics/AbcSize
-          params = {per_page: batch_size(options)}.merge(params)
-
-          launch(
-            data,
-            method:  :get,
-            params:  params,
-            silent:  silent,
-            options: options
-          ).tap do |status|
-            status.mode = :search
-
-            entries = status.queue
-            puts "\n"
-            entries.each_with_index do |entry, i|
-              if (i % 10).zero?
-                percent = i * 100 / entries.length
-                print "Searching: #{percent.round}% (#{i}/#{entries.length} entries)\r"
-                $stdout.flush
-              end
-
-              next if status.success?(entry)
-
-              email = nil
-              if entry.respond_to?(:email)
-                email = entry.email
-              elsif entry.respond_to?(:to_h)
-                email = entry.to_h["email"]
-              end
-
-              people_matching = []
-              email = email.to_s.strip.downcase
-              unless email.empty?
-                people_matching = get(
-                  params:  params.merge(q: email),
-                  silent:  silent,
-                  options: options
-                ).select do |person|
-                  person.email == email
-                end
-              end
-
-              case people_matching.length
-              when 1
-                status.set_person_match(entry, people_matching.first)
-              when 2..Float::INFINITY
-                status.set_people_match(entry, people_matching)
-              end
-            end
-          end
-        end
-
-        private
-
-        def get(params: {}, silent: false, options: self.options)
-          msg = "cannot batch get without api connnection, please provide a valid api connection!"
-          fatal msg unless (people_api = api&.people)
-
-          params = {per_page: batch_size(options)}.merge(params)
-          people_api.get_all(params: params, silent: silent)
-        end
-
-        def batch_from(
-          data,
-          method:,
-          params:  {},
-          silent:  false,
-          options: self.options
-        )
-          fatal "Invalid batch method: #{method}." unless self.class.valid_method?(method)
-          return nil if !data || !data.is_a?(Enumerable)
-
-          msg = "cannot batch #{method} without api connnection, please provide a valid api connection!"
-          fatal msg unless (people_api = api&.people)
-
-          # param q does not make sense here, even for GET method
-          params   = {per_page: batch_size(options)}.merge(params)
-          per_page = params[:per_page] || batch_size(options)
-
-          launch_batch(
-            data,
-            method:     method,
-            per_page:   per_page,
-            people_api: people_api,
-            silent:     silent,
-            options:    options
-          )
-        end
-
-        # Default way to retrieve options (unless provided)
-        def options
-          ASSETS.cli.options
-        end
-
-        def launch_batch( # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
-          data,
-          method:,
-          status:     nil,
-          job_mode:   true, # rubocop:disable Lint/UnusedMethodArgument
-          options:    self.options,
-          per_page:   batch_size(options),
-          people_api: api&.people,
-          silent:     false
-        )
-          iteration  = 1
-          done       = 0
-          iterations = (data.length.to_f / per_page).ceil
-
-          status ||= Eco::API::Session::Batch::Status.new(
-            enviro,
-            queue:  data,
-            method: method
-          )
-
-          status.tap do
-            pending_for_server_error = data.to_a[0..]
-
-            start_time = Time.now
-
-            data.each_slice(per_page) do |slice|
-              msg  = "starting batch '#{method}' iteration #{iteration}/#{iterations}, "
-              msg << "with #{slice.length} entries of #{data.length} -- #{done} done"
-              msg << (" " * 20)
-              log(:info) { msg } unless silent
-
-              start_slice = Time.now
-
-              offer_retry_on(Ecoportal::API::Errors::TimeOut) do
-                people_api.batch(job_mode: job_mode?(options)) do |batch|
-                  slice.each do |person|
-                    batch.public_send(method, person) do |response|
-                      faltal("Request with no response") unless response
-
-                      next if server_error?(response)
-
-                      pending_for_server_error.delete(person)
-                      status[person] = response
-                    end
-                  end
-                end # end batch
-              end
-
-              done += slice.length
-
-              msg  = "  ... iteration #{iteration}/#{iterations} done "
-              msg << "in #{str_stats(start_slice, slice.length)} "
-              msg << "(average: #{str_stats(start_time, done)})"
-              msg << (" " * 20)
-              log(:info) { msg } unless silent
-
-              iteration  += 1
-            end # next slice
-
-            # temporary working around (due to back-end problems with batch/jobs)
-            unless pending_for_server_error.empty?
-              msg = "Going to re-try #{pending_for_server_error.count} due to server errors"
-              log(:info) { msg } unless silent
-
-              launch_batch(
-                pending_for_server_error,
-                status:     status,
-                method:     method,
-                job_mode:   false,
-                per_page:   per_page,
-                people_api: people_api,
-                silent:     silent,
-                options:    options
-              )
-            end
-          end
-        end
-
-        def server_error?(response)
-          res_status   = response.status
-          server_error = !res_status || res_status.server_error?
-          other_error  = !server_error && (!res_status.code || res_status.code < 100)
-          no_body      = !server_error && !other_error && !response.body
-          server_error || other_error || no_body
-        end
-
-        def offer_retry_on(error_type, retries_left = 3, &block)
-          yield
-        rescue error_type => err
-          raise err.class, err.message, cause: nil unless retries_left.positive?
-
-          explanation = "#{err}\n"
-          explanation << "You have #{retries_left} retries left."
-          question    = " Do you want to retry (y/N)?"
-
-          prompt_user(question, default: "Y", explanation: explanation, timeout: 10) do |response|
-            raise unless response.upcase.start_with?("Y")
-
-            puts "\nOkay... let's retry!"
-            offer_retry_on(error_type, retries_left - 1, &block)
-          end
-        end
-
-        def str_stats(start, count)
-          now     = Time.now
-          secs    = (now - start).round(2)
-          if secs > 0.0
-            per_sec = (count.to_f / secs).round(2)
-            "#{secs}s -> #{per_sec} people/s"
-          else
-            " -- "
-          end
+        def launch(...)
+          batch_from(...)
         end
       end
     end

data/lib/eco/version.rb

@@ -1,3 +1,3 @@
 module Eco
-  VERSION = '3.0.16'.freeze
+  VERSION = '3.0.18'.freeze
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: eco-helpers
 version: !ruby/object:Gem::Version
-  version: 3.0.16
+  version: 3.0.18
 platform: ruby
 authors:
 - Oscar Segura
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2024-10-14 00:00:00.000000000 Z
+date: 2024-10-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -229,7 +229,7 @@ dependencies:
         version: '0.10'
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.10.4
+        version: 0.10.6
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -239,7 +239,7 @@ dependencies:
         version: '0.10'
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.10.4
+        version: 0.10.6
 - !ruby/object:Gem::Dependency
   name: ecoportal-api-graphql
   requirement: !ruby/object:Gem::Requirement
@@ -269,7 +269,7 @@ dependencies:
         version: '2.0'
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.0.10
+        version: 2.0.11
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -279,7 +279,7 @@ dependencies:
         version: '2.0'
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.0.10
+        version: 2.0.11
 - !ruby/object:Gem::Dependency
   name: ed25519
   requirement: !ruby/object:Gem::Requirement
@@ -656,8 +656,16 @@ files:
 - lib/eco/api/session/batch/job.rb
 - lib/eco/api/session/batch/jobs.rb
 - lib/eco/api/session/batch/jobs_groups.rb
+- lib/eco/api/session/batch/launcher.rb
+- lib/eco/api/session/batch/launcher/benchmarking.rb
+- lib/eco/api/session/batch/launcher/mode_size.rb
+- lib/eco/api/session/batch/launcher/options.rb
+- lib/eco/api/session/batch/launcher/retry.rb
+- lib/eco/api/session/batch/launcher/status_handling.rb
+- lib/eco/api/session/batch/launcher/valid_methods.rb
 - lib/eco/api/session/batch/policies.rb
 - lib/eco/api/session/batch/request_stats.rb
+- lib/eco/api/session/batch/searcher.rb
 - lib/eco/api/session/batch/status.rb
 - lib/eco/api/session/config.rb
 - lib/eco/api/session/config/api.rb