ecoportal-api 0.10.2 → 0.10.4

data/lib/ecoportal/api/common/client/with_retry.rb

@@ -0,0 +1,86 @@
+module Ecoportal
+  module API
+    module Common
+      class Client
+        module WithRetry
+          DELAY_REQUEST_RETRY       = 5
+          RETRY_ATTEMPTS            = 5
+          HANDLED_CONNECTION_ERRORS = [
+            HTTP::ConnectionError,
+            IOError
+          ].freeze
+
+          include Ecoportal::API::Common::Client::ElasticApmIntegration
+
+          private
+
+          # Helper to ensure unexpected server errors do not bring
+          # client scripts immediately down
+          # @note it manages limited range of errors, the rest
+          #   are not handled.
+          def with_retry(
+            attempts = retry_attemps,
+            delay    = delay_request_retry,
+            error_safe: true,
+            &block
+          )
+            response = nil
+
+            attempts.times do |i|
+              remaining = attempts - i - 1
+
+              response = with_connection_error_handling(
+                remaining,
+                error_safe: error_safe,
+                callback:   block
+              ) do
+                block.call
+              end
+
+              return response unless some_unexpected_error?(response)
+
+              # handle server errors (5xx) & server bugs (i.e. empty body)
+              msg  = "re-attempting (remaining: "
+              msg << "#{remaining} attempts out of #{attempts})"
+              log(:debug) { msg }
+
+              log_unexpected_server_error(response)
+
+              msg  = "Got server error (#{response.status}): #{response.body}\n"
+              msg << "Going to retry (##{i} of #{attempts})"
+              log(:debug) { msg }
+
+              sleep(delay) if i < attempts
+            end
+
+            response
+          end
+
+          def with_connection_error_handling(remaining, callback:, error_safe: true)
+            yield
+          rescue *handled_connection_errors => err
+            raise unless error_safe && remaining.positive?
+
+            msg = "Got #{err.class}: #{err.message}"
+            log(:debug) { msg }
+
+            with_retry(remaining, error_safe: error_safe, &callback)
+          end
+
+          # Add here other connection errors
+          def handled_connection_errors
+            self.class::HANDLED_CONNECTION_ERRORS
+          end
+
+          def retry_attemps
+            self.class::RETRY_ATTEMPTS
+          end
+
+          def delay_request_retry
+            self.class::DELAY_REQUEST_RETRY
+          end
+        end
+      end
+    end
+  end
+end

data/lib/ecoportal/api/common/client.rb

@@ -1,4 +1,12 @@
 require 'http'
+
+require 'ecoportal/api/common/client/error'
+require 'ecoportal/api/common/client/elastic_apm_integration'
+require 'ecoportal/api/common/client/rate_throttling'
+require 'ecoportal/api/common/client/throughput'
+require 'ecoportal/api/common/client/time_out'
+require 'ecoportal/api/common/client/with_retry'
+
 module Ecoportal
   module API
     module Common
@@ -16,9 +24,10 @@ module Ecoportal
       # @attr_reader logger [Logger] the logger.
       # @attr_reader host [String] the remote target server.
       class Client
-        include Common::ElasticApmIntegration
-        DELAY_REQUEST_RETRY = 5
-        RETRY_ATTEMPTS      = 5
+        include WithRetry
+        include RateThrottling
+
+        DEFAULT_HOST = 'live.ecoportal.com'.freeze
 
         attr_accessor :logger
         attr_reader :host
@@ -28,40 +37,32 @@ module Ecoportal
         # @param version [String] it is part of the base url and will determine the api version we query against.
         # @param host [String] api server domain.
         # @param logger [Logger] an object with `Logger` interface to generate logs.
-        # @param response_logging [Boolean] whether or not batch responses should be logged
+        # @param deep_logging [Boolean] whether or not batch responses should be logged
         # @return [Client] an object that holds the configuration of the api connection.
-        def initialize(api_key:, version: "v1", host: "live.ecoportal.com", logger: nil, response_logging: false)
-          @version  = version
-          @api_key  = api_key
-          @logger   = logger
-          @host     = host
-          @response_logging_enabled = response_logging
+        def initialize(api_key:, version: "v1", host: DEFAULT_HOST, logger: nil, deep_logging: false)
+          @version      = version
+          @api_key      = api_key
+          @logger       = logger
+          @host         = host
+          @deep_logging = deep_logging
 
           if host.match(/^localhost|^127\.0\.0\.1/)
             @base_uri = "http://#{host}/api/"
           else
             @base_uri = "https://#{host}/api/"
           end
-          log(:info) { "#{version} client initialized pointing at #{host}" }
+
+          if deep_logging?
+            log(:debug) {
+              "#{version} client initialized pointing at #{host}"
+            }
+          end
 
           return unless @api_key.nil? || @api_key.match(/\A\W*\z/)
 
           log(:error) { "Api-key missing!" }
         end
 
-        # Logger interface.
-        # @example:
-        #   log(:info) {"General information on what's going on"}
-        #   log(:warn) {"This is a warning that something is likely to have gone amiss"}
-        #   log(:error) {"Something went wrong"}
-        #   log(:fatal) {"An unrecoverable error has happend"}
-        # @param level [Symbol] the level that the message should be logged.
-        # @yield [] generates the message.
-        # @yieldreturn [String] the generated message.
-        def log(level, &block)
-          logger&.send(level, &block)
-        end
-
         # Sends an http `GET` request against the api version using `path` to complete the base url,
         # and adding the key_value pairs of `params` in the http _header_.
         # @param path [String] the tail that completes the url of the request.
@@ -152,7 +153,7 @@ module Ecoportal
         # @param path [String] the tail that completes the url of the request.
         # @return [String] the final url.
         def url_for(path)
-          @base_uri+@version+path
+          "#{@base_uri}#{@version}#{path}"
         end
 
         private
@@ -161,71 +162,45 @@ module Ecoportal
           raise "Expected block" unless block_given?
 
           start_time = Time.now.to_f
-          log(:info)  { "#{method} #{url_for(path)}" }
-          log(:debug) { "Data: #{JSON.pretty_generate(data)}" }
 
-          with_retry(&block).tap do |result|
+          if deep_logging?
+            log(:debug) { "#{method} #{url_for(path)}" }
+            log(:debug) { "Data: #{JSON.pretty_generate(data)}" }
+          end
+
+          with_retry do
+            rate_throttling(&block)
+          end.tap do |result|
+            next unless deep_logging?
+
             end_time = Time.now.to_f
-            log(result.success?? :info : :warn) {
+
+            log(:debug) {
               "Took %.2fs, Status #{result.status}" % (end_time - start_time) # rubocop:disable Style/FormatString
             }
 
-            next unless @response_logging_enabled
-
-            log(result.success?? :debug : :warn) {
+            log(:debug) {
               "Response: #{JSON.pretty_generate(result.body)}"
             }
           end
         end
 
-        # Helper to ensure unexpected server errors do not bring client scripts immediately down
-        def with_retry(attempts = RETRY_ATTEMPTS, delay = DELAY_REQUEST_RETRY, error_safe: true, &block)
-          response = nil
-          attempts.times do |i|
-            remaining = attempts - i - 1
-
-            begin
-              response = block.call
-            rescue HTTP::ConnectionError => e
-              raise unless error_safe && remaining.positive?
-              log(:error) { "Got connection error: #{e.message}" }
-              response = with_retry(remaining, error_safe: error_safe, &block)
-            rescue IOError => e
-              raise unless error_safe && remaining.positive?
-              log(:error) { "Got IO error: #{e.message}" }
-              response = with_retry(remaining, error_safe: error_safe, &block)
-            end
-
-            return response unless some_unexpected_error?(response)
-
-            puts "re-attempting (remaining: #{remaining} attempts out of #{attempts})"
-
-            log_unexpected_server_error(response)
-
-            msg  = "Got server error (#{response.status}): #{response.body}\n"
-            msg += "Going to retry (#{i} out of #{attempts})"
-            log(:error) { msg }
-
-            sleep(delay) if i < attempts
-          end
-          response
-        end
-
-        # Sometimes response body is wrong but status code
-        # doesn't reflect. Let it retry
-        def some_unexpected_error?(response)
-          unexpected_server_error?(response.status) ||
-            unexpected_body?(response)
+        def deep_logging?
+          @deep_logging
         end
 
-        def unexpected_body?(response)
-          response.body.nil?.tap do |wrong|
-            next unless wrong
-
-            msg  = "Received non json body in response "
-            msg << "(#{response.src_body.class}):\n #{response.src_body}"
-            puts
-          end
+        # Logger interface.
+        # @example:
+        #   log(:info) {"General information on what's going on"}
+        #   log(:warn) {"This is a warning that something is likely to have gone amiss"}
+        #   log(:error) {"Something went wrong"}
+        #   log(:fatal) {"An unrecoverable error has happend"}
+        # @param level [Symbol] the level that the message should be logged.
+        # @yield [] generates the message.
+        # @yieldreturn [String] the generated message.
+        def log(level, &block)
+          puts "(#{level}) #{yield}"
+          logger&.send(level, &block)
         end
       end
     end

data/lib/ecoportal/api/common.rb

@@ -10,8 +10,6 @@ require 'ecoportal/api/common/hash_diff'
 require 'ecoportal/api/common/base_model'
 require 'ecoportal/api/common/doc_helpers'
 require 'ecoportal/api/common/logging'
-require 'ecoportal/api/common/elastic_apm_integration'
-require 'ecoportal/api/common/time_out'
 require 'ecoportal/api/common/client'
 require 'ecoportal/api/common/response'
 require 'ecoportal/api/common/wrapped_response'

data/lib/ecoportal/api/v1/job/awaiter.rb

@@ -0,0 +1,115 @@
+module Ecoportal
+  module API
+    class V1
+      class Job
+        class Awaiter
+          include Common::Client::TimeOut
+
+          DELAY_STATUS_CHECK = 4
+          MIN_STATUS_CHECK   = 2
+
+          TIMEOUT_APPROACH   = :conservative # adaptative timeout
+          TIMEOUT_FALLBACK   = :min
+
+          attr_reader :job, :job_id, :total
+          attr_accessor :timeout_approach
+
+          def initialize(job, job_id:, total:)
+            @job     = job
+            @job_id  = job_id
+            @total   = total
+            @checked = false
+            self.timeout_approach = self.class::TIMEOUT_APPROACH
+          end
+
+          # Allows to preserve the learned throughput
+          def new(**kargs)
+            self.class.new(job, **kargs).tap do |out|
+              out.throughput = throughput
+            end
+          end
+
+          def await_completion! # rubocop:disable Metrics/AbcSize
+            max_timeout = timeout_for(total, approach: timeout_approach)
+
+            # timeout library is evil. So we make poor-man timeout.
+            # https://jvns.ca/blog/2015/11/27/why-rubys-timeout-is-dangerous-and-thread-dot-raise-is-terrifying/
+            before = Time.now
+            delay_status_check = nil
+
+            loop do
+              status  = job.status(job_id)
+              waited  = Time.now - before
+
+              adapted = waited
+              adapted = waited - (delay_status_check / 2) if delay_status_check
+              ratio   = throughput!(adapted, count: status.progress)
+
+              break status if status.complete?(total)
+
+              pending = status.pending(total)
+              left    = max_timeout - waited
+
+              timeout!(status, timeout: max_timeout) unless left.positive?
+
+              delay_status_check = status_check_in(pending, timeout_in: left)
+
+              msg  = " ... Awaiting #{delay_status_check} s. -- "
+              msg << " TimeOut: #{left.round(2)} s. "
+              msg << "(job '#{job_id}') "
+              msg << "Done: #{status.progress} (est. #{ratio} rec/s) "
+              msg << "     \r"
+
+              print msg
+              $stdout.flush
+
+              sleep(delay_status_check)
+            end
+          end
+
+          private
+
+          def timeout!(status, timeout:)
+            self.timeout_approach = self.class::TIMEOUT_FALLBACK
+
+            msg  = "Job '#{job_id}' not complete (size: #{total}).\n"
+            msg << "  Timed out after #{timeout} seconds.\n"
+            msg << "  Current status: #{status}"
+
+            raise API::Errors::TimeOut, msg
+          end
+
+          def checked?
+            @checked
+          end
+
+          def status_check_in(pending, timeout_in:)
+            unless checked?
+              @checked = true
+              return min_delay_status_check
+            end
+
+            return default_delay_status_check if around_min_throughput?
+
+            eta           = eta_for(pending, approach: :optimistic)
+            check_in_max  = [eta, timeout_in].min * 0.90
+
+            check_in_best = check_in_max / 2.0
+            default_5     = default_delay_status_check * 5
+
+            top_check     = [default_5, check_in_best].min.ceil
+            [top_check, min_delay_status_check].max
+          end
+
+          def default_delay_status_check
+            self.class::DELAY_STATUS_CHECK
+          end
+
+          def min_delay_status_check
+            self.class::MIN_STATUS_CHECK
+          end
+        end
+      end
+    end
+  end
+end

data/lib/ecoportal/api/v1/job/status.rb

@@ -0,0 +1,42 @@
+module Ecoportal
+  module API
+    class V1
+      class Job
+        class Status
+          attr_reader :id, :progress
+
+          def initialize(id, complete, errored, progress)
+            @id       = id
+            @complete = complete
+            @errored  = errored
+            @progress = progress
+          end
+
+          def complete?(total = nil)
+            return @complete if total.nil?
+
+            progress >= total
+          end
+
+          def pending(total)
+            return 1 unless total
+            return 0 if total <= progress
+
+            total - progress
+          end
+
+          def errored?
+            @errored
+          end
+
+          def to_s
+            msg = complete? ? "Completed" : "In progress"
+            msg = "Errored" if errored?
+            msg << " with #{progress} done."
+            msg
+          end
+        end
+      end
+    end
+  end
+end

data/lib/ecoportal/api/v1/job.rb

@@ -0,0 +1,131 @@
+require 'ecoportal/api/v1/job/status'
+require 'ecoportal/api/v1/job/awaiter'
+
+module Ecoportal
+  module API
+    class V1
+      class Job
+        attr_reader :client, :person_class
+
+        # @param client [Common::Client] a `Common::Client` object that holds the configuration of the api connection.
+        # @return [People] an instance object ready to make people api requests.
+        def initialize(client, person_class:)
+          @client       = client
+          @person_class = person_class
+          @created      = false
+        end
+
+        # Allows to preserve the learned througoutput
+        def new
+          self.class.new(client, person_class: person_class).tap do |out|
+            out.awaiter = @awaiter
+          end
+        end
+
+        def created?
+          @created
+        end
+
+        # @return [Ecoportal::API::Common::Response] the results of the batch job
+        def batch(recover: false)
+          raise_if_already_launched! unless recover
+
+          @operation ||= Common::BatchOperation.new(
+            "/people",
+            person_class,
+            logger: client.logger
+          )
+
+          yield operation unless recover
+
+          total   = operation.count
+          job_id  = create(operation, recover: recover)
+          stat    = awaiter(
+            job_id: job_id,
+            total:  total
+          ).await_completion!
+
+          job_result(job_id, operation) if stat&.complete?(total)
+        end
+
+        def status(job_id)
+          response = client.get("/people/job/#{CGI.escape(job_id)}/status")
+          body     = response && body_data(response.body)
+
+          msg      = "Status error (#{response.status}) - "
+          msg     << "Errors: #{body}"
+          raise msg unless response.success?
+
+          Status.new(*body.values_at(*%w[id complete errored progress]))
+        end
+
+        protected
+
+        attr_writer :awaiter
+
+        def awaiter(job_id:, total:, preserve: true)
+          return @awaiter = Awaiter.new(self, job_id: job_id, total: total) unless preserve && @awaiter
+
+          @awaiter = @awaiter.new(job_id: job_id, total: total)
+        end
+
+        private
+
+        attr_reader :operation
+
+        def operations
+          @operations ||= {}
+        end
+
+        def job_operation(job_id)
+          operations[job_id]
+        end
+
+        def raise_if_already_launched!
+          return unless created?
+
+          msg  = "Missusage: job was already created."
+          msg << " Can't call batch more than once"
+          raise msg
+        end
+
+        # @return [Ecoportal::API::Common::Response] the results of the batch job
+        def job_result(job_id, operation)
+          client.get("/people/job/#{CGI.escape(job_id)}").tap do |response|
+            operation.process_response(response)
+          end
+        end
+
+        # @return [String] the `id` of the created batch job
+        def create(operation, recover: false)
+          raise_if_already_launched! unless recover
+
+          job_id = nil
+
+          client.post("/people/job", data: operation.as_json).tap do |response|
+            job_id = body_data(response.body)["id"] if response.success?
+
+            next if job_id
+
+            msg  = "Could not create job - Error (#{response.status}): "
+            msg << body_data(response.body).to_s
+            raise msg
+          end
+
+          if job_id
+            @created = true
+            operations[job_id] = operation
+          end
+
+          job_id
+        end
+
+        # Hook for other api versions to obtain the raw data of a response
+        # @note this was introduced to allow `v2` to reuse this class
+        def body_data(body)
+          body
+        end
+      end
+    end
+  end
+end

data/lib/ecoportal/api/v1/people.rb

@@ -6,11 +6,8 @@ module Ecoportal
       class People
         extend Common::BaseClass
         include Common::DocHelpers
-        include Common::TimeOut
         include Enumerable
 
-        DELAY_STATUS_CHECK = 5
-
         class_resolver :person_class, "Ecoportal::API::V1::Person"
 
         attr_reader :client
@@ -143,7 +140,7 @@ module Ecoportal
         # @param job_mode [Boolean] whether or not it should use batch jobs
         # @return [Ecoportal::API::Common::Response] the results of the batch
         def batch(job_mode: true, &block)
-          return job(&block) if job_mode
+          return job.batch(&block) if job_mode
 
           operation = Common::BatchOperation.new(
             "/people",
@@ -159,33 +156,11 @@ module Ecoportal
           end
         end
 
-        # @return [Ecoportal::API::Common::Response] the results of the batch job
-        def job
-          operation = Common::BatchOperation.new(
-            "/people",
-            person_class,
-            logger: client.logger
-          )
-
-          yield operation
-
-          total   = operation.count
-          timeout = timeout_for(total)
-
-          job_id  = create_job(operation)
-          status  = wait_for_job_completion(job_id, timeout: timeout, total: total)
+        # @return [Ecoportal::API::V1::Job]
+        def job(preserve_stats: true)
+          return @last_job = Job.new(client, person_class: person_class) unless preserve_stats && @last_job
 
-          # @todo
-          # if total == status.progress
-          if status&.complete?(total)
-            job_result(job_id, operation)
-          else
-            msg  = "Job '#{job_id}' not complete (size: #{total}).\n"
-            msg << "  Probably timeout after #{timeout} seconds.\n"
-            msg << "  Current status: #{status}"
-
-            raise API::Errors::TimeOut, msg
-          end
+          @last_job = @last_job.new
         end
 
         # Creates a new `Person` object.
@@ -196,59 +171,6 @@ module Ecoportal
 
         private
 
-        def job_status(job_id)
-          response = client.get("/people/job/#{CGI.escape(job_id)}/status")
-          body     = response && body_data(response.body)
-
-          msg      = "Status error (#{response.status}) - "
-          msg     << "Errors: #{body}"
-          raise msg unless response.success?
-
-          JobStatus.new(*body.values_at(*%w[id complete errored progress]))
-        end
-
-        # @return [Ecoportal::API::Common::Response] the results of the batch job
-        def job_result(job_id, operation)
-          client.get("/people/job/#{CGI.escape(job_id)}").tap do |response|
-            operation.process_response(response)
-          end
-        end
-
-        def wait_for_job_completion(job_id, timeout:, total:)
-          # timeout library is evil. So we make poor-man timeout.
-          # https://jvns.ca/blog/2015/11/27/why-rubys-timeout-is-dangerous-and-thread-dot-raise-is-terrifying/
-          before = Time.now
-
-          loop do
-            status = job_status(job_id)
-            break status if status.complete?(total)
-
-            left = (before + timeout) - Time.now
-            break status unless left.positive?
-            # break status if Time.now >= before + timeout
-
-            msg  = " ... Await job "
-            msg << "('#{job_id}'; done: #{status.progress}): "
-            msg << "#{left.ceil} sec.    \r"
-
-            print msg
-            $stdout.flush
-
-            sleep(DELAY_STATUS_CHECK)
-            status
-          end
-        end
-
-        # @return [String] the `id` of the created batch job
-        def create_job(operation)
-          job_id = nil
-          client.post("/people/job", data: operation.as_json).tap do |response|
-            job_id = body_data(response.body)["id"] if response.success?
-            raise "Could not create job - Error (#{response.status}): #{body_data(response.body)}" unless job_id
-          end
-          job_id
-        end
-
         # Hook for other api versions to obtain the raw data of a response
         # @note this was introduced to allow `v2` to reuse this class
         def body_data(body)
@@ -262,3 +184,4 @@ end
 require 'ecoportal/api/v1/schema_field_value'
 require 'ecoportal/api/v1/person_details'
 require 'ecoportal/api/v1/person'
+require 'ecoportal/api/v1/job'