eco-helpers 2.0.42 → 2.0.43

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c14b952724b755aa88b633094baf947b8c32b3a8699f77e8a4d63cdc94e10765
-  data.tar.gz: 015fd112ec97cb490c1bae82cdbd744256ca8879b98578f4c2442a0730652bf5
+  metadata.gz: dc85b6b42cc78adcdf2cba976331b5ae0202464ade71bb9f6b72793c28c4e8f0
+  data.tar.gz: 3ab9745bb62f78df1bdd6ec9657607d4135cce910c643fb9adedbb42b315e022
 SHA512:
-  metadata.gz: cbdf02bb1b3323069682c6b9dca370b598d846fc185e8db6dba7e337f1939a41f00b17d60bcb599c4f8abf9c0c6a713495e88258eb5a521b061ffa01c63d552d
-  data.tar.gz: b656b2a52dc65a7e2ae582ec68999de0d7551a6a17c5b83a3763ead94d161a8c969dc76add52ec6a93ebbfd68b5b47293c06fe3e33c8cfaacc4ad6273bf5b17f
+  metadata.gz: 339e768dd7de68dd4f87743003bb1d177220a7ac47808b19cbcf837f16a75e1d7c3f65ce427e67542e0fe318be8b2b2531f5bd0e99743bd049c03c1711965aa0
+  data.tar.gz: 26b8d2e9a17e8ae7ad447ea3c28a9ce02d1726c0d78839175fc1a432c5c8aff63dde66a2bcec4bca94c745cdd794e5a0caf3a5e25008e159a3c29ead2fd707c9

data/CHANGELOG.md

@@ -1,6 +1,17 @@
 # Change Log
 All notable changes to this project will be documented in this file.
 
+## [2.0.43] - 2021-11-25
+
+### Added
+  - `Eco::API::Session::Batch::Job` added better **logging**
+
+### Changed
+  - `Eco::API::Session::Batch` added **resilience** and **recovery** to connection errors
+  - `Eco::API::Policies::DefaultPolicies::UserAccess` changed logging from `warn` to `info`.
+
+### Fixed
+
 ## [2.0.42] - 2021-10-30
 
 ### Added

data/lib/eco/api/policies/default_policies/99_user_access_policy.rb

@@ -22,7 +22,7 @@ class Eco::API::Policies::DefaultPolicies::UserAccess < Eco::API::Common::Loader
   def warn_account_removal!
     if account_removed_count > 0
       msg = "(DefaultPolicy on job '#{job.name}') Removed account to #{account_removed_count} people"
-      session.logger.warn(msg)
+      session.logger.info(msg)
     end
   end
 

data/lib/eco/api/session/batch/job.rb

@@ -176,6 +176,7 @@ module Eco
               if pqueue.length > 0
                 req_backup = as_update(pqueue, add_feedback: false)
                 backup_update(req_backup)
+                logger.debug("Job ('#{name}':#{type}): going to launch batch against #{pqueue.count} entries")
                 session.batch.launch(pqueue, method: type).tap do |job_status|
                   @status     = job_status
                   status.root = self
@@ -334,7 +335,7 @@ module Eco
                 handlers.each do |handler|
                   if entries = err_types[handler.name]
                     handler_job = subjobs_add("#{self.name} => #{handler.name}", usecase: handler)
-                    logger.debug("Running error handler #{handler.name}")
+                    logger.debug("Running error handler #{handler.name} (against #{entries.count} entries)")
                     handler.launch(people: people(entries), session: session, options: options, job: handler_job)
                     logger.debug("Launching job of error handler: #{handler_job.name}")
                     handler_job.launch(simulate: simulate)

data/lib/eco/api/session/batch.rb

@@ -92,6 +92,7 @@ module Eco
           return people_api.get_all(params: params, silent: silent)
         end
 
+
         def batch_from(data, method:, params: {}, silent: false)
           fatal "Invalid batch method: #{method}." if !self.class.valid_method?(method)
           return nil if !data || !data.is_a?(Enumerable)
@@ -101,12 +102,23 @@ module Eco
           params   = {per_page: DEFAULT_BATCH_BLOCK}.merge(params)
           per_page = params[:per_page] || DEFAULT_BATCH_BLOCK
 
+          launch_batch(data,
+            method: method,
+            per_page: per_page,
+            people_api: people_api,
+            silent: silent
+          )
+        end
+
+        def launch_batch(data, method:, status: nil, job_mode: true, per_page: DEFAULT_BATCH_BLOCK, people_api: api&.people, silent: false)
           iteration  = 1; done = 0
           iterations = (data.length.to_f / per_page).ceil
 
-          Eco::API::Session::Batch::Status.new(enviro, queue: data, method: method).tap do |status|
+          status ||= Eco::API::Session::Batch::Status.new(enviro, queue: data, method: method)
+          status.tap do |status|
             start_time  = Time.now
             start_slice = Time.now; slice = []
+            pending_for_server_error = data.to_a[0..-1]
             data.each_slice(per_page) do |slice|
               msg  = "starting batch '#{method}' iteration #{iteration}/#{iterations},"
               msg += " with #{slice.length} entries of #{data.length} -- #{done} done"
@@ -115,11 +127,14 @@ module Eco
 
               start_slice = Time.now
               offer_retry_on(Ecoportal::API::Errors::TimeOut) do
-                people_api.batch do |batch|
+                people_api.batch(job_mode: false) do |batch|
                   slice.each do |person|
                     batch.public_send(method, person) do |response|
                       faltal("Request with no response") unless !!response
-                      status[person] = response
+                      unless server_error?(response)
+                        pending_for_server_error.delete(person)
+                        status[person] = response
+                      end
                     end
                   end
                 end # end batch
@@ -128,9 +143,31 @@ module Eco
               iteration  += 1
               done       += slice.length
             end # next slice
+
+            # temporary working around (due to back-end problems with batch/jobs)
+            unless pending_for_server_error.empty?
+              msg = "Going to re-try #{pending_for_server_error.count} due to server errors"
+              logger.info(msg) unless silent
+              launch_batch(pending_for_server_error,
+                status: status,
+                method: method,
+                job_mode: false,
+                per_page: per_page,
+                people_api: people_api,
+                silent: silent
+              )
+            end
           end
         end
 
+        def server_error?(response)
+          res_status   = response.status
+          server_error = !res_status || res_status.server_error?
+          other_error  = !server_error && (!res_status.code || res_status.code < 100)
+          no_body      = !server_error && !other_error  && !response.body
+          server_error || other_error || no_body
+        end
+
         def offer_retry_on(error_type, retries_left = 3, &block)
           begin
             block.call

data/lib/eco/version.rb

@@ -1,3 +1,3 @@
 module Eco
-  VERSION = "2.0.42"
+  VERSION = "2.0.43"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: eco-helpers
 version: !ruby/object:Gem::Version
-  version: 2.0.42
+  version: 2.0.43
 platform: ruby
 authors:
 - Oscar Segura