ecoportal-api 0.10.1 → 0.10.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 84d530d29757e28159e985e70e8a0da65f16d831ea2658fcafbe94f42e98f882
-  data.tar.gz: a4778b377de727e3c648fa79d674b044b32d452c7cc4f024d5fbb383185a3097
+  metadata.gz: dde26458f8ba991809f2218f6a4594101350f861a811d1bf4c6b508b2194ef67
+  data.tar.gz: f9bd67ceaea38ec6ea82d9a53ae0651c36fb7f74e417b37029acd7d72fd00d97
 SHA512:
-  metadata.gz: cb0dad40ddd3d5b0b621ff61b063bf2e92522b09156035009a15e782d946d43aa9db28c0066c3db1c805c78f29a2ee550fd7c358020b7216a776624c3d31f43e
-  data.tar.gz: c84332d17ddd19fbfbbd69f287c2110cad88064ebce801f232fcaeeccc5e6db5ec2c5ce359267d43df6bd8f4bf08238b0c242f30ef39789b92321030a914b09f
+  metadata.gz: '09dd4b2cde2bd8f6c6689433c7472fedb6110d076603afcd8edc39f5275aed0296470e57900eaceb335dc875f31554d515925c6b1eae060786a055ac706f6a66'
+  data.tar.gz: 3363216e775f38601146617ee80abab13625163c46739f536d77f94c17247fcce139331cad8af02b4cbcc0848a352211a34c476d0d766082e18b5d4b222f43cf

data/.rubocop.yml

@@ -30,6 +30,11 @@ Style/ConditionalAssignment:
 Style/BlockDelimiters:
   BracesRequiredMethods: ['log']
   AllowedPatterns: ['proc', 'new']
+Style/HashSyntax:
+  EnforcedShorthandSyntax: either
+  EnforcedStyle: no_mixed_keys
+Style/ArgumentsForwarding:
+  UseAnonymousForwarding: false
 Style/ClassAndModuleChildren:
   Enabled: false
 Style/FrozenStringLiteralComment:
@@ -93,3 +98,5 @@ Naming/MethodParameterName:
   AllowedNames: ['x', 'y', 'i', 'j', 'id', 'io', 'to']
 Naming/RescuedExceptionsVariableName:
   Enabled: false
+Naming/BlockForwarding:
+  Enabled: false

data/CHANGELOG.md

@@ -2,19 +2,45 @@
 
 All notable changes to this project will be documented in this file.
 
+## [0.10.4] - 2024-10-xx
+
+### Added
+
+### Changed
+
+### Fixed
+
+## [0.10.3] - 2024-10-01
+
+### Changed
+
+- refactored Client
+  - initialized with `deep_logging` named argumnet
+  - previously `response_logging`
+
+## [0.10.2] - 2024-09-27
+
+### Added
+
+- `Ecoportal::API::Common::TimeOut` to calculate adaptive time outs.
+
+### Changed
+
+- `Ecoportal::API::V1::Person#job`
+  - added **adaptative await**
+  - evaluate `true` to `complete?` if `progress` is that of total count.
+
 ## [0.10.1] - 2024-08-01
 
 ### Added
 
-- `Ecoportal::V1::Person#brand_id`
-- `Ecoportal::V1::Person#archived`
+- `Ecoportal::API::V1::Person#brand_id`
+- `Ecoportal::API::V1::Person#archived`
 
 ### Changed
 
 - require `ruby 3`
 
-### Fixed
-
 ## [0.9.8] - 2024-05-15
 
 ### Added
@@ -344,5 +370,3 @@ All notable changes to this project will be documented in this file.
 
 - this `CHANGELOG.md` file
 - person model: `freemium` core property
-
-

data/ecoportal-api.gemspec

@@ -1,12 +1,17 @@
+# rubocop:disable Gemspec/DevelopmentDependencies
 lib = File.expand_path('lib', __dir__)
 $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
-require "ecoportal/api/version"
+
+require 'ecoportal/api/version'
 
 Gem::Specification.new do |spec|
   spec.name          = "ecoportal-api"
   spec.version       = Ecoportal::API::VERSION
-  spec.authors       = ["Tapio Saarinen"]
-  spec.email         = ["tapio@ecoportal.co.nz", "rien@ecoportal.co.nz", "oscar@ecoportal.co.nz", "bozydar@ecoportal.co.nz"]
+  spec.authors       = ['Tapio Saarinen']
+  spec.email         = [
+    'tapio@ecoportal.co.nz',
+    'oscar@ecoportal.co.nz'
+  ]
 
   spec.summary       = %q{A collection of helpers for interacting with the ecoPortal MS's various APIs}
   spec.homepage      = "https://www.ecoportal.com"
@@ -19,17 +24,21 @@ Gem::Specification.new do |spec|
   spec.files = `git ls-files -z`.split("\x0").reject do |f|
     f.match(%r{^(test|spec|features)/})
   end
-  spec.bindir        = "exe"
+  spec.bindir        = 'exe'
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
-  spec.require_paths = ["lib"]
+  spec.require_paths = ['lib']
 
-  spec.add_development_dependency "rspec",     ">= 3.12.0", "< 4"
-  spec.add_development_dependency "rake",      ">= 13.1.0", "< 14"
-  spec.add_development_dependency "yard",      ">= 0.9.34", "< 1"
-  spec.add_development_dependency "redcarpet", ">= 3.6.0",  "< 4"
-  spec.add_development_dependency "pry"      , "~> 0.14"
+  spec.add_development_dependency 'pry'      , '~> 0.14'
+  spec.add_development_dependency 'rake',      '>= 13.0.3', '< 14'
+  spec.add_development_dependency 'redcarpet', '>= 3.6.0',  '< 4'
+  spec.add_development_dependency 'rspec',     '>= 3.12.0', '< 4'
+  spec.add_development_dependency 'rubocop',   '~> 1'
+  spec.add_development_dependency 'rubocop-rake', '~> 0'
+  spec.add_development_dependency 'yard',      '~> 0.9'
 
   spec.add_dependency 'dotenv',        '~> 3'
   spec.add_dependency 'elastic-apm',   '>= 4.7',   "< 5"
   spec.add_dependency 'http',          '~> 5.1',   "< 6"
 end
+
+# rubocop:enable Gemspec/DevelopmentDependencies

data/lib/ecoportal/api/common/batch_operation.rb

@@ -4,11 +4,16 @@ module Ecoportal
       class BatchOperation
         include Common::DocHelpers
 
-        def initialize(base_path, wrapper, logger: nil)
-          @base_path  = base_path
-          @wrapper    = wrapper
-          @operations = []
-          @logger     = logger
+        def initialize(base_path, wrapper, logger: nil, deep_logging: false)
+          @base_path    = base_path
+          @wrapper      = wrapper
+          @operations   = []
+          @logger       = logger
+          @deep_logging = deep_logging
+        end
+
+        def count
+          @operations.count
         end
 
         def as_json
@@ -21,11 +26,12 @@ module Ecoportal
 
         def process_response(response)
           unless response.success?
-            log(:error) { "Total failure in batch operation." }
-            raise "Total failure in batch operation"
+            msg = "Error: total failure in batch operation."
+            log(:debug) { msg }
+            raise msg
           end
 
-          log(:info) { "Processing batch responses" }
+          log(:debug) { "Processing batch responses" } if deep_logging?
 
           body_data(response.body).each.with_index do |subresponse, idx|
             status   = subresponse["status"]
@@ -106,14 +112,19 @@ module Ecoportal
         end
 
         def log_batch_response(operation, response)
-          log(:info) { "BATCH #{operation[:method]} #{operation[:path]}" }
-          log(:info) { "Status #{response.status}" }
+          return unless deep_logging?
+
+          log(:debug) { "BATCH #{operation[:method]} #{operation[:path]}" }
+          log(:debug) { "Status #{response.status}" }
+          log(:debug) { "Response: #{JSON.pretty_generate(response.body)}" }
+        end
 
-          level = response.success?? :debug : :warn
-          log(level) { "Response: #{JSON.pretty_generate(response.body)}" }
+        def deep_logging?
+          @deep_logging
         end
 
         def log(level, &block)
+          puts "(#{level}) #{yield}"
           @logger&.send(level, &block)
         end
       end

data/lib/ecoportal/api/common/client/elastic_apm_integration.rb

@@ -0,0 +1,111 @@
+require 'elastic-apm'
+module Ecoportal
+  module API
+    module Common
+      class Client
+        module ElasticApmIntegration
+          include Ecoportal::API::Common::Client::Error::Checks
+
+          APM_SERVICE_NAME = 'ecoportal-api-gem'.freeze
+
+          # Log only errors that are only server's responsibility
+          def log_unexpected_server_error(response)
+            msg = "Expecting Ecoportal::API::Common::Response. Given: #{response.class}"
+            raise msg unless response.is_a?(Common::Response)
+
+            return unless elastic_apm_service
+            return unless unexpected_server_error_code?(response.status)
+            return unless ElasticAPM.running?
+
+            ElasticAPM.report(
+              Ecoportal::API::Common::Client::Error::UnexpectedServerError.new(
+                response.body,
+                code: response.status
+              )
+            )
+          end
+
+          private
+
+          # finalizer to stop the agent
+          close_elastic_apm = proc do |_id|
+            next unless ElasticAPM.running?
+
+            puts "Stopping ElasticAPM service"
+            ElasticAPM.stop
+          rescue StandardError
+            # Silent
+          end
+
+          ObjectSpace.define_finalizer("ElasticAPM", close_elastic_apm)
+
+          def elastic_apm_service
+            return false if @disable_apm
+
+            ElasticAPM.start(**elastic_apm_options) unless ElasticAPM.running?
+          rescue StandardError => err
+            @disable_apm = true
+            puts "ElasticAPM services not available: #{err}"
+          end
+
+          def elastic_apm_options
+            {
+              service_name:             APM_SERVICE_NAME,
+              server_url:               elastic_apm_url,
+              secret_token:             elastic_apm_key,
+              environment:              environment,
+              # http_compression:        false,
+              transaction_sample_rate:  0.1,
+              transaction_max_spans:    100,
+              span_frames_min_duration: "5ms"
+            }.tap do |options|
+              # next unless false
+
+              options.merge!({
+                log_level: Logger::DEBUG,
+                log_path:  File.join(__dir__, "elastic_apm.log")
+              })
+            end
+          end
+
+          def elastic_apm_url
+            @elastic_apm_url ||= "https://".tap do |url|
+              url << elastic_apm_account_id.to_s
+              url << ".#{elastic_apm_base_url}"
+              url << ":#{elastic_apm_port}"
+            end
+          end
+
+          def elastic_apm_key
+            @elastic_apm_key ||= ENV['ELASTIC_APM_KEY']
+          end
+
+          def elastic_apm_account_id
+            @elastic_apm_account_id ||= ENV['ELASTIC_APM_ACCOUNT_ID']
+          end
+
+          def elastic_apm_base_url
+            @elastic_apm_base_url ||= "apm.#{elastic_apm_region}.aws.cloud.es.io"
+          end
+
+          def elastic_apm_region
+            @elastic_apm_region ||= ENV['ELASTIC_APM_REGION'] || "ap-southeast-2"
+          end
+
+          def elastic_apm_port
+            @elastic_apm_port ||= ENV['ELASTIC_APM_PORT'] || "443"
+          end
+
+          def environment
+            @environment ||= "unknown".tap do |value|
+              next unless instance_variable_defined?(:@host)
+              next unless (env = @host.gsub(".ecoportal.com", ''))
+
+              value.clear << env
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/ecoportal/api/common/client/error/checks.rb

@@ -0,0 +1,39 @@
+module Ecoportal
+  module API
+    module Common
+      class Client
+        module Error
+          module Checks
+            private
+
+            def unexpected_server_error_code?(code)
+              return true unless code
+              return true if (code >= 500) && (code <= 599)
+
+              code <= 99
+            end
+
+            # Sometimes response body is wrong but status code
+            # doesn't reflect. Let it retry
+            def some_unexpected_error?(response)
+              return true if unexpected_server_error_code?(response.status)
+
+              unexpected_body?(response)
+            end
+
+            def unexpected_body?(response)
+              response.body.nil?.tap do |wrong|
+                next unless wrong
+
+                msg  = "Received non json body in response "
+                msg << "(#{response.src_body.class}):\n "
+                msg << response.src_body
+                puts
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/ecoportal/api/common/client/error.rb

@@ -0,0 +1,17 @@
+require 'ecoportal/api/common/client/error/checks'
+
+module Ecoportal
+  module API
+    module Common
+      class Client
+        module Error
+          class UnexpectedServerError < StandardError
+            def initialize(msg, code:)
+              super("Code: #{code} -- Error: #{msg}")
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/ecoportal/api/common/client/time_out.rb

@@ -0,0 +1,28 @@
+module Ecoportal
+  module API
+    module Common
+      class Client
+        module TimeOut
+          MIN_THROUGHPUT = 0.2 # people per second
+          MIN_SIZE       = 10
+
+          private
+
+          def min_throughput
+            self.class::MIN_THROUGHPUT
+          end
+
+          def min_size
+            self.class::MIN_SIZE
+          end
+
+          def timeout_for(count)
+            count = 1        unless count&.positive?
+            count = min_size if count < min_size
+            (count.ceil / min_throughput).ceil
+          end
+        end
+      end
+    end
+  end
+end

data/lib/ecoportal/api/common/client/with_retry.rb

@@ -0,0 +1,86 @@
+module Ecoportal
+  module API
+    module Common
+      class Client
+        module WithRetry
+          DELAY_REQUEST_RETRY       = 5
+          RETRY_ATTEMPTS            = 5
+          HANDLED_CONNECTION_ERRORS = [
+            HTTP::ConnectionError,
+            IOError
+          ].freeze
+
+          include Ecoportal::API::Common::Client::ElasticApmIntegration
+
+          private
+
+          # Helper to ensure unexpected server errors do not bring
+          # client scripts immediately down
+          # @note it manages limited range of errors, the rest
+          #   are not handled.
+          def with_retry(
+            attempts = retry_attemps,
+            delay    = delay_request_retry,
+            error_safe: true,
+            &block
+          )
+            response = nil
+
+            attempts.times do |i|
+              remaining = attempts - i - 1
+
+              response = with_connection_error_handling(
+                remaining,
+                error_safe: error_safe,
+                callback:   block
+              ) do
+                block.call
+              end
+
+              return response unless some_unexpected_error?(response)
+
+              # handle server errors (5xx) & server bugs (i.e. empty body)
+              msg  = "re-attempting (remaining: "
+              msg << "#{remaining} attempts out of #{attempts})"
+              log(:debug) { msg }
+
+              log_unexpected_server_error(response)
+
+              msg  = "Got server error (#{response.status}): #{response.body}\n"
+              msg << "Going to retry (##{i} of #{attempts})"
+              log(:debug) { msg }
+
+              sleep(delay) if i < attempts
+            end
+
+            response
+          end
+
+          def with_connection_error_handling(remaining, callback:, error_safe: true)
+            yield
+          rescue *handled_connection_errors => err
+            raise unless error_safe && remaining.positive?
+
+            msg = "Got #{err.class}: #{err.message}"
+            log(:debug) { msg }
+
+            with_retry(remaining, error_safe: error_safe, &callback)
+          end
+
+          # Add here other connection errors
+          def handled_connection_errors
+            self.class::HANDLED_CONNECTION_ERRORS
+          end
+
+          def retry_attemps
+            self.class::RETRY_ATTEMPTS
+          end
+
+          def delay_request_retry
+            self.class::DELAY_REQUEST_RETRY
+          end
+        end
+      end
+    end
+  end
+end

data/lib/ecoportal/api/common/client.rb

@@ -1,4 +1,10 @@
 require 'http'
+
+require 'ecoportal/api/common/client/error'
+require 'ecoportal/api/common/client/elastic_apm_integration'
+require 'ecoportal/api/common/client/time_out'
+require 'ecoportal/api/common/client/with_retry'
+
 module Ecoportal
   module API
     module Common
@@ -16,9 +22,7 @@ module Ecoportal
       # @attr_reader logger [Logger] the logger.
       # @attr_reader host [String] the remote target server.
       class Client
-        include Common::ElasticApmIntegration
-        DELAY_REQUEST_RETRY = 5
-        RETRY_ATTEMPTS      = 5
+        include WithRetry
 
         attr_accessor :logger
         attr_reader :host
@@ -28,40 +32,32 @@ module Ecoportal
         # @param version [String] it is part of the base url and will determine the api version we query against.
         # @param host [String] api server domain.
         # @param logger [Logger] an object with `Logger` interface to generate logs.
-        # @param response_logging [Boolean] whether or not batch responses should be logged
+        # @param deep_logging [Boolean] whether or not batch responses should be logged
         # @return [Client] an object that holds the configuration of the api connection.
-        def initialize(api_key:, version: "v1", host: "live.ecoportal.com", logger: nil, response_logging: false)
-          @version  = version
-          @api_key  = api_key
-          @logger   = logger
-          @host     = host
-          @response_logging_enabled = response_logging
+        def initialize(api_key:, version: "v1", host: "live.ecoportal.com", logger: nil, deep_logging: false)
+          @version      = version
+          @api_key      = api_key
+          @logger       = logger
+          @host         = host
+          @deep_logging = deep_logging
 
           if host.match(/^localhost|^127\.0\.0\.1/)
             @base_uri = "http://#{host}/api/"
           else
             @base_uri = "https://#{host}/api/"
           end
-          log(:info) { "#{version} client initialized pointing at #{host}" }
+
+          if deep_logging?
+            log(:debug) {
+              "#{version} client initialized pointing at #{host}"
+            }
+          end
 
           return unless @api_key.nil? || @api_key.match(/\A\W*\z/)
 
           log(:error) { "Api-key missing!" }
         end
 
-        # Logger interface.
-        # @example:
-        #   log(:info) {"General information on what's going on"}
-        #   log(:warn) {"This is a warning that something is likely to have gone amiss"}
-        #   log(:error) {"Something went wrong"}
-        #   log(:fatal) {"An unrecoverable error has happend"}
-        # @param level [Symbol] the level that the message should be logged.
-        # @yield [] generates the message.
-        # @yieldreturn [String] the generated message.
-        def log(level, &block)
-          logger&.send(level, &block)
-        end
-
         # Sends an http `GET` request against the api version using `path` to complete the base url,
         # and adding the key_value pairs of `params` in the http _header_.
         # @param path [String] the tail that completes the url of the request.
@@ -152,7 +148,7 @@ module Ecoportal
         # @param path [String] the tail that completes the url of the request.
         # @return [String] the final url.
         def url_for(path)
-          @base_uri+@version+path
+          "#{@base_uri}#{@version}#{path}"
         end
 
         private
@@ -161,71 +157,43 @@ module Ecoportal
           raise "Expected block" unless block_given?
 
           start_time = Time.now.to_f
-          log(:info)  { "#{method} #{url_for(path)}" }
-          log(:debug) { "Data: #{JSON.pretty_generate(data)}" }
+
+          if deep_logging?
+            log(:debug) { "#{method} #{url_for(path)}" }
+            log(:debug) { "Data: #{JSON.pretty_generate(data)}" }
+          end
 
           with_retry(&block).tap do |result|
+            next unless deep_logging?
+
             end_time = Time.now.to_f
-            log(result.success?? :info : :warn) {
+
+            log(:debug) {
               "Took %.2fs, Status #{result.status}" % (end_time - start_time) # rubocop:disable Style/FormatString
             }
 
-            next unless @response_logging_enabled
-
-            log(result.success?? :debug : :warn) {
+            log(:debug) {
               "Response: #{JSON.pretty_generate(result.body)}"
             }
           end
         end
 
-        # Helper to ensure unexpected server errors do not bring client scripts immediately down
-        def with_retry(attempts = RETRY_ATTEMPTS, delay = DELAY_REQUEST_RETRY, error_safe: true, &block)
-          response = nil
-          attempts.times do |i|
-            remaining = attempts - i - 1
-
-            begin
-              response = block.call
-            rescue HTTP::ConnectionError => e
-              raise unless error_safe && remaining.positive?
-              log(:error) { "Got connection error: #{e.message}" }
-              response = with_retry(remaining, error_safe: error_safe, &block)
-            rescue IOError => e
-              raise unless error_safe && remaining.positive?
-              log(:error) { "Got IO error: #{e.message}" }
-              response = with_retry(remaining, error_safe: error_safe, &block)
-            end
-
-            return response unless some_unexpected_error?(response)
-
-            puts "re-attempting (remaining: #{remaining} attempts out of #{attempts})"
-
-            log_unexpected_server_error(response)
-
-            msg  = "Got server error (#{response.status}): #{response.body}\n"
-            msg += "Going to retry (#{i} out of #{attempts})"
-            log(:error) { msg }
-
-            sleep(delay) if i < attempts
-          end
-          response
-        end
-
-        # Sometimes response body is wrong but status code
-        # doesn't reflect. Let it retry
-        def some_unexpected_error?(response)
-          unexpected_server_error?(response.status) ||
-            unexpected_body?(response)
+        def deep_logging?
+          @deep_logging
         end
 
-        def unexpected_body?(response)
-          response.body.nil?.tap do |wrong|
-            next unless wrong
-
-            msg  = "Received non json body in response "
-            msg << "(#{response.src_body.class}):\n #{response.src_body}"
-            puts
-          end
+        # Logger interface.
+        # @example:
+        #   log(:info) {"General information on what's going on"}
+        #   log(:warn) {"This is a warning that something is likely to have gone amiss"}
+        #   log(:error) {"Something went wrong"}
+        #   log(:fatal) {"An unrecoverable error has happend"}
+        # @param level [Symbol] the level that the message should be logged.
+        # @yield [] generates the message.
+        # @yieldreturn [String] the generated message.
+        def log(level, &block)
+          puts "(#{level}) #{yield}"
+          logger&.send(level, &block)
         end
       end
     end

data/lib/ecoportal/api/common/doc_helpers.rb

@@ -12,7 +12,7 @@ module Ecoportal
             doc
           end
         end
-        
+
         def get_id(doc)
           id = nil
           id ||= doc.id if doc.respond_to?(:id)

data/lib/ecoportal/api/common.rb

@@ -10,7 +10,6 @@ require 'ecoportal/api/common/hash_diff'
 require 'ecoportal/api/common/base_model'
 require 'ecoportal/api/common/doc_helpers'
 require 'ecoportal/api/common/logging'
-require 'ecoportal/api/common/elastic_apm_integration'
 require 'ecoportal/api/common/client'
 require 'ecoportal/api/common/response'
 require 'ecoportal/api/common/wrapped_response'

data/lib/ecoportal/api/v1/job_status.rb

@@ -0,0 +1,33 @@
+module Ecoportal
+  module API
+    class V1
+      class JobStatus
+        attr_reader :id, :progress
+
+        def initialize(id, complete, errored, progress)
+          @id       = id
+          @complete = complete
+          @errored  = errored
+          @progress = progress
+        end
+
+        def complete?(total = nil)
+          return @complete if total.nil?
+
+          progress >= total
+        end
+
+        def errored?
+          @errored
+        end
+
+        def to_s
+          msg = complete? ? "Completed" : "In progress"
+          msg = "Errored" if errored?
+          msg << " with #{progress} done."
+          msg
+        end
+      end
+    end
+  end
+end

data/lib/ecoportal/api/v1/people.rb

@@ -6,9 +6,9 @@ module Ecoportal
       class People
         extend Common::BaseClass
         include Common::DocHelpers
+        include Common::Client::TimeOut
         include Enumerable
 
-        JOB_TIMEOUT        = 240
         DELAY_STATUS_CHECK = 5
 
         class_resolver :person_class, "Ecoportal::API::V1::Person"
@@ -169,15 +169,20 @@ module Ecoportal
 
           yield operation
 
-          job_id = create_job(operation)
-          status = wait_for_job_completion(job_id)
+          total   = operation.count
+          timeout = timeout_for(total)
 
-          if status&.complete?
+          job_id  = create_job(operation)
+          status  = wait_for_job_completion(job_id, timeout: timeout, total: total)
+
+          # @todo
+          # if total == status.progress
+          if status&.complete?(total)
             job_result(job_id, operation)
           else
-            msg  = "Job `#{job_id}` not complete. "
-            msg << "Probably timeout after #{JOB_TIMEOUT} seconds. "
-            msg << "Current status: #{status}"
+            msg  = "Job '#{job_id}' not complete (size: #{total}).\n"
+            msg << "  Probably timeout after #{timeout} seconds.\n"
+            msg << "  Current status: #{status}"
 
             raise API::Errors::TimeOut, msg
           end
@@ -191,8 +196,6 @@ module Ecoportal
 
         private
 
-        JobStatus = Struct.new(:id, :complete?, :errored?, :progress)
-
         def job_status(job_id)
           response = client.get("/people/job/#{CGI.escape(job_id)}/status")
           body     = response && body_data(response.body)
@@ -201,12 +204,7 @@ module Ecoportal
           msg     << "Errors: #{body}"
           raise msg unless response.success?
 
-          JobStatus.new(
-            body["id"],
-            body["complete"],
-            body["errored"],
-            body["progress"]
-          )
+          JobStatus.new(*body.values_at(*%w[id complete errored progress]))
         end
 
         # @return [Ecoportal::API::Common::Response] the results of the batch job
@@ -216,15 +214,25 @@ module Ecoportal
           end
         end
 
-        def wait_for_job_completion(job_id)
+        def wait_for_job_completion(job_id, timeout:, total:)
           # timeout library is evil. So we make poor-man timeout.
           # https://jvns.ca/blog/2015/11/27/why-rubys-timeout-is-dangerous-and-thread-dot-raise-is-terrifying/
           before = Time.now
 
           loop do
             status = job_status(job_id)
-            break status if status.complete?
-            break status if Time.now >= before + JOB_TIMEOUT
+            break status if status.complete?(total)
+
+            left = (before + timeout) - Time.now
+            break status unless left.positive?
+            # break status if Time.now >= before + timeout
+
+            msg  = " ... Await job "
+            msg << "('#{job_id}'; done: #{status.progress}): "
+            msg << "#{left.ceil} sec.    \r"
+
+            print msg
+            $stdout.flush
 
             sleep(DELAY_STATUS_CHECK)
             status

data/lib/ecoportal/api/v1.rb

@@ -46,5 +46,6 @@ module Ecoportal
   end
 end
 
+require 'ecoportal/api/v1/job_status'
 require 'ecoportal/api/v1/person_schemas'
 require 'ecoportal/api/v1/people'

data/lib/ecoportal/api/version.rb

@@ -1,5 +1,5 @@
 module Ecoportal
   module API
-    VERSION = "0.10.1".freeze
+    VERSION = '0.10.3'.freeze
   end
 end

metadata

@@ -1,42 +1,36 @@
 --- !ruby/object:Gem::Specification
 name: ecoportal-api
 version: !ruby/object:Gem::Version
-  version: 0.10.1
+  version: 0.10.3
 platform: ruby
 authors:
 - Tapio Saarinen
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2024-08-01 00:00:00.000000000 Z
+date: 2024-09-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: rspec
+  name: pry
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 3.12.0
-    - - "<"
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '4'
+        version: '0.14'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 3.12.0
-    - - "<"
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '4'
+        version: '0.14'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 13.1.0
+        version: 13.0.3
     - - "<"
       - !ruby/object:Gem::Version
         version: '14'
@@ -46,37 +40,37 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 13.1.0
+        version: 13.0.3
     - - "<"
       - !ruby/object:Gem::Version
         version: '14'
 - !ruby/object:Gem::Dependency
-  name: yard
+  name: redcarpet
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.9.34
+        version: 3.6.0
     - - "<"
       - !ruby/object:Gem::Version
-        version: '1'
+        version: '4'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.9.34
+        version: 3.6.0
     - - "<"
       - !ruby/object:Gem::Version
-        version: '1'
+        version: '4'
 - !ruby/object:Gem::Dependency
-  name: redcarpet
+  name: rspec
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 3.6.0
+        version: 3.12.0
     - - "<"
       - !ruby/object:Gem::Version
         version: '4'
@@ -86,24 +80,52 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 3.6.0
+        version: 3.12.0
     - - "<"
       - !ruby/object:Gem::Version
         version: '4'
 - !ruby/object:Gem::Dependency
-  name: pry
+  name: rubocop
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.14'
+        version: '1'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.14'
+        version: '1'
+- !ruby/object:Gem::Dependency
+  name: rubocop-rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: yard
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.9'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.9'
 - !ruby/object:Gem::Dependency
   name: dotenv
   requirement: !ruby/object:Gem::Requirement
@@ -161,9 +183,7 @@ dependencies:
 description:
 email:
 - tapio@ecoportal.co.nz
-- rien@ecoportal.co.nz
 - oscar@ecoportal.co.nz
-- bozydar@ecoportal.co.nz
 executables: []
 extensions: []
 extra_rdoc_files: []
@@ -189,8 +209,12 @@ files:
 - lib/ecoportal/api/common/batch_operation.rb
 - lib/ecoportal/api/common/batch_response.rb
 - lib/ecoportal/api/common/client.rb
+- lib/ecoportal/api/common/client/elastic_apm_integration.rb
+- lib/ecoportal/api/common/client/error.rb
+- lib/ecoportal/api/common/client/error/checks.rb
+- lib/ecoportal/api/common/client/time_out.rb
+- lib/ecoportal/api/common/client/with_retry.rb
 - lib/ecoportal/api/common/doc_helpers.rb
-- lib/ecoportal/api/common/elastic_apm_integration.rb
 - lib/ecoportal/api/common/hash_diff.rb
 - lib/ecoportal/api/common/logging.rb
 - lib/ecoportal/api/common/response.rb
@@ -215,6 +239,7 @@ files:
 - lib/ecoportal/api/internal/schema_field_value.rb
 - lib/ecoportal/api/logger.rb
 - lib/ecoportal/api/v1.rb
+- lib/ecoportal/api/v1/job_status.rb
 - lib/ecoportal/api/v1/people.rb
 - lib/ecoportal/api/v1/person.rb
 - lib/ecoportal/api/v1/person_details.rb
@@ -243,7 +268,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.5.6
+rubygems_version: 3.5.18
 signing_key:
 specification_version: 4
 summary: A collection of helpers for interacting with the ecoPortal MS's various APIs

data/lib/ecoportal/api/common/elastic_apm_integration.rb

@@ -1,112 +0,0 @@
-require 'elastic-apm'
-module Ecoportal
-  module API
-    module Common
-      module ElasticApmIntegration
-
-        class UnexpectedServerError < StandardError
-          def initialize(code, msg)
-            super("Code: #{code} -- Error: #{msg}")
-          end
-        end
-
-        APM_SERVICE_NAME = 'ecoportal-api-gem'
-
-        # Log only errors that are only server's responsibility
-        def log_unexpected_server_error(response)
-          raise "Expecting Ecoportal::API::Common::Response. Given: #{response.class}" unless response.is_a?(Common::Response)
-          return nil unless elastic_apm_service
-          return nil unless unexpected_server_error?(response.status)
-          if ElasticAPM.running?
-            ElasticAPM.report(UnexpectedServerError.new(response.status, response.body))
-          end
-        end
-
-        private
-
-        def unexpected_server_error?(code)
-          !code || ((code >= 500) && (code <= 599)) || (code <= 99)
-        end
-
-        # finalizer to stop the agent
-        close_elastic_apm = Proc.new do |id|
-          begin
-            if ElasticAPM.running?
-              puts "Stopping ElasticAPM service"
-              ElasticAPM.stop
-            end
-          rescue StandardError => e
-            # Silent
-          end
-        end
-        ObjectSpace.define_finalizer("ElasticAPM", close_elastic_apm)
-
-        def elastic_apm_service
-          return false if @disable_apm
-          begin
-            ElasticAPM.start(**elastic_apm_options) unless ElasticAPM.running?
-          rescue StandardError => e
-            @disable_apm = true
-            puts "ElasticAPM services not available: #{e}"
-          end
-        end
-
-        def elastic_apm_options
-          {
-            service_name: APM_SERVICE_NAME,
-            server_url:   elastic_apm_url,
-            secret_token: elastic_apm_key,
-            environment:  environment,
-            #http_compression: false,
-            transaction_sample_rate:  0.1,
-            transaction_max_spans:    100,
-            span_frames_min_duration: "5ms"
-          }.tap do |options|
-            options.merge!({
-              log_level: Logger::DEBUG,
-              log_path:  File.join(__dir__, "elastic_apm.log")
-            }) if false
-          end
-        end
-
-        def elastic_apm_url
-          @elastic_apm_url ||= "https://".tap do |url|
-            url << "#{elastic_apm_account_id}"
-            url << ".#{elastic_apm_base_url}"
-            url << ":#{elastic_apm_port}"
-          end
-        end
-
-        def elastic_apm_key
-          @elastic_apm_key ||= ENV['ELASTIC_APM_KEY']
-        end
-
-        def elastic_apm_account_id
-          @elastic_apm_account_id ||= ENV['ELASTIC_APM_ACCOUNT_ID']
-        end
-
-        def elastic_apm_base_url
-          @elastic_apm_base_url ||= "apm.#{elastic_apm_region}.aws.cloud.es.io"
-        end
-
-        def elastic_apm_region
-          @elastic_apm_region ||= ENV['ELASTIC_APM_REGION'] || "ap-southeast-2"
-        end
-
-
-        def elastic_apm_port
-          @elastic_apm_port ||= ENV['ELASTIC_APM_PORT'] || "443"
-        end
-
-        def environment
-          @environment ||= "unknown".tap do |value|
-            if instance_variable_defined?(:@host) && env = @host.gsub(".ecoportal.com", '')
-              value.clear << env
-            end
-          end
-        end
-
-      end
-    end
-  end
-end