ecoportal-api 0.10.2 → 0.10.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4cc0547ca88715ad6b0f2d3bc7ed4661bab0f9259117ce28aa5d43aab0761d02
-  data.tar.gz: 1e109b427a0e71bd91edf1d4e9ce6e7cf9ac2df998662c97746df30b8e07998b
+  metadata.gz: 28e9d05f838bf52d038a6d9533a8bcbb98f7b66b5ffa6fa7e99a4c30b2d0e486
+  data.tar.gz: a6586e72297c5e109ed4f2e8a04beb32e1e435ea3eb0332803b4ba9fd91e8fb3
 SHA512:
-  metadata.gz: 44eba3743e30beff3c04e71b1191900b8477d81af1371f1a5003cb6847384e384d535e51390e4f8db39e8aa81a0cbf7fc200a87ac18036c45e56530b72b09342
-  data.tar.gz: 1749045310b0748e8935027cd0d1b9d69e7a84c4534b27017bd1581848d579c6359e59dedfe19e3cbe451038e2a4c70fb82150a01ca4c9260eadde64bfac5a92
+  metadata.gz: b8ecb9337ed3b2f11913ae5823bdb3b4771d1bcb364a50afc68bba38518dff39fd7986de641a4030030cfeeb0019a5960d9ac2440a10ee2bb49ed15aad77f0a8
+  data.tar.gz: 1f3ca204cb6c8b4a6cab2396285bf50644cb955f1120c15253f1d81ac1663bf2874f2b3706f5dedcfbdc34bf7c67d5a0520f1e62c0e8891259b337d92394cbc3

data/CHANGELOG.md

@@ -2,7 +2,7 @@
 
 All notable changes to this project will be documented in this file.
 
-## [0.10.3] - 2024-09-xx
+## [0.10.5] - 2024-10-xx
 
 ### Added
 
@@ -10,6 +10,29 @@ All notable changes to this project will be documented in this file.
 
 ### Fixed
 
+## [0.10.4] - 2024-10-15
+
+### Added
+
+- `Ecoportal::API::Common::Client::RateThrottling`
+  - See: <https://github.com/zombocom/rate_throttle_client?tab=readme-ov-file#ratethrottleclient>
+  - Add adapter to the RateLimit, may it be present.
+  - **Default:** `:proportional_decrease`
+
+### Changed
+
+- `Ecoportal::API::Common::Client::TimeOut`
+  - Const `MIN_SIZE` changed from `10` to `5` to speed up small requests.
+- Full revamp of `Job` endpoint
+
+## [0.10.3] - 2024-10-01
+
+### Changed
+
+- refactored Client
+  - initialized with `deep_logging` named argumnet
+  - previously `response_logging`
+
 ## [0.10.2] - 2024-09-27
 
 ### Added

data/ecoportal-api.gemspec

@@ -13,7 +13,7 @@ Gem::Specification.new do |spec|
     'oscar@ecoportal.co.nz'
   ]
 
-  spec.summary       = %q{A collection of helpers for interacting with the ecoPortal MS's various APIs}
+  spec.summary       = "A collection of helpers for interacting with the ecoPortal MS's various APIs"
   spec.homepage      = "https://www.ecoportal.com"
   spec.licenses      = %w[MIT]
 
@@ -28,17 +28,18 @@ Gem::Specification.new do |spec|
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ['lib']
 
-  spec.add_development_dependency 'pry'      , '~> 0.14'
+  spec.add_development_dependency 'pry',       '~> 0.14'
   spec.add_development_dependency 'rake',      '>= 13.0.3', '< 14'
   spec.add_development_dependency 'redcarpet', '>= 3.6.0',  '< 4'
   spec.add_development_dependency 'rspec',     '>= 3.12.0', '< 4'
   spec.add_development_dependency 'rubocop',   '~> 1'
   spec.add_development_dependency 'rubocop-rake', '~> 0'
-  spec.add_development_dependency 'yard',      '~> 0.9'
+  spec.add_development_dependency 'yard',         '~> 0.9'
 
   spec.add_dependency 'dotenv',        '~> 3'
   spec.add_dependency 'elastic-apm',   '>= 4.7',   "< 5"
   spec.add_dependency 'http',          '~> 5.1',   "< 6"
+  spec.add_dependency 'rate_throttle_client', '~> 0.1'
 end
 
 # rubocop:enable Gemspec/DevelopmentDependencies

data/lib/ecoportal/api/common/batch_operation.rb

@@ -4,11 +4,12 @@ module Ecoportal
       class BatchOperation
         include Common::DocHelpers
 
-        def initialize(base_path, wrapper, logger: nil)
-          @base_path  = base_path
-          @wrapper    = wrapper
-          @operations = []
-          @logger     = logger
+        def initialize(base_path, wrapper, logger: nil, deep_logging: false)
+          @base_path    = base_path
+          @wrapper      = wrapper
+          @operations   = []
+          @logger       = logger
+          @deep_logging = deep_logging
         end
 
         def count
@@ -25,11 +26,12 @@ module Ecoportal
 
         def process_response(response)
           unless response.success?
-            log(:error) { "Total failure in batch operation." }
-            raise "Total failure in batch operation"
+            msg = "Error: total failure in batch operation."
+            log(:debug) { msg }
+            raise msg
           end
 
-          log(:info) { "Processing batch responses" }
+          log(:debug) { "Processing batch responses" } if deep_logging?
 
           body_data(response.body).each.with_index do |subresponse, idx|
             status   = subresponse["status"]
@@ -110,14 +112,19 @@ module Ecoportal
         end
 
         def log_batch_response(operation, response)
-          log(:info) { "BATCH #{operation[:method]} #{operation[:path]}" }
-          log(:info) { "Status #{response.status}" }
+          return unless deep_logging?
 
-          level = response.success?? :debug : :warn
-          log(level) { "Response: #{JSON.pretty_generate(response.body)}" }
+          log(:debug) { "BATCH #{operation[:method]} #{operation[:path]}" }
+          log(:debug) { "Status #{response.status}" }
+          log(:debug) { "Response: #{JSON.pretty_generate(response.body)}" }
+        end
+
+        def deep_logging?
+          @deep_logging
         end
 
         def log(level, &block)
+          puts "(#{level}) #{yield}"
           @logger&.send(level, &block)
         end
       end

data/lib/ecoportal/api/common/client/elastic_apm_integration.rb

@@ -0,0 +1,111 @@
+require 'elastic-apm'
+module Ecoportal
+  module API
+    module Common
+      class Client
+        module ElasticApmIntegration
+          include Ecoportal::API::Common::Client::Error::Checks
+
+          APM_SERVICE_NAME = 'ecoportal-api-gem'.freeze
+
+          # Log only errors that are only server's responsibility
+          def log_unexpected_server_error(response)
+            msg = "Expecting Ecoportal::API::Common::Response. Given: #{response.class}"
+            raise msg unless response.is_a?(Common::Response)
+
+            return unless elastic_apm_service
+            return unless unexpected_server_error_code?(response.status)
+            return unless ElasticAPM.running?
+
+            ElasticAPM.report(
+              Ecoportal::API::Common::Client::Error::UnexpectedServerError.new(
+                response.body,
+                code: response.status
+              )
+            )
+          end
+
+          private
+
+          # finalizer to stop the agent
+          close_elastic_apm = proc do |_id|
+            next unless ElasticAPM.running?
+
+            puts "Stopping ElasticAPM service"
+            ElasticAPM.stop
+          rescue StandardError
+            # Silent
+          end
+
+          ObjectSpace.define_finalizer("ElasticAPM", close_elastic_apm)
+
+          def elastic_apm_service
+            return false if @disable_apm
+
+            ElasticAPM.start(**elastic_apm_options) unless ElasticAPM.running?
+          rescue StandardError => err
+            @disable_apm = true
+            puts "ElasticAPM services not available: #{err}"
+          end
+
+          def elastic_apm_options
+            {
+              service_name:             APM_SERVICE_NAME,
+              server_url:               elastic_apm_url,
+              secret_token:             elastic_apm_key,
+              environment:              environment,
+              # http_compression:        false,
+              transaction_sample_rate:  0.1,
+              transaction_max_spans:    100,
+              span_frames_min_duration: "5ms"
+            }.tap do |options|
+              # next unless false
+
+              options.merge!({
+                log_level: Logger::DEBUG,
+                log_path:  File.join(__dir__, "elastic_apm.log")
+              })
+            end
+          end
+
+          def elastic_apm_url
+            @elastic_apm_url ||= "https://".tap do |url|
+              url << elastic_apm_account_id.to_s
+              url << ".#{elastic_apm_base_url}"
+              url << ":#{elastic_apm_port}"
+            end
+          end
+
+          def elastic_apm_key
+            @elastic_apm_key ||= ENV['ELASTIC_APM_KEY']
+          end
+
+          def elastic_apm_account_id
+            @elastic_apm_account_id ||= ENV['ELASTIC_APM_ACCOUNT_ID']
+          end
+
+          def elastic_apm_base_url
+            @elastic_apm_base_url ||= "apm.#{elastic_apm_region}.aws.cloud.es.io"
+          end
+
+          def elastic_apm_region
+            @elastic_apm_region ||= ENV['ELASTIC_APM_REGION'] || "ap-southeast-2"
+          end
+
+          def elastic_apm_port
+            @elastic_apm_port ||= ENV['ELASTIC_APM_PORT'] || "443"
+          end
+
+          def environment
+            @environment ||= "unknown".tap do |value|
+              next unless instance_variable_defined?(:@host)
+              next unless (env = @host.gsub(".ecoportal.com", ''))
+
+              value.clear << env
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/ecoportal/api/common/client/error/checks.rb

@@ -0,0 +1,39 @@
+module Ecoportal
+  module API
+    module Common
+      class Client
+        module Error
+          module Checks
+            private
+
+            def unexpected_server_error_code?(code)
+              return true unless code
+              return true if (code >= 500) && (code <= 599)
+
+              code <= 99
+            end
+
+            # Sometimes response body is wrong but status code
+            # doesn't reflect. Let it retry
+            def some_unexpected_error?(response)
+              return true if unexpected_server_error_code?(response.status)
+
+              unexpected_body?(response)
+            end
+
+            def unexpected_body?(response)
+              response.body.nil?.tap do |wrong|
+                next unless wrong
+
+                msg  = "Received non json body in response "
+                msg << "(#{response.src_body.class}):\n "
+                msg << response.src_body
+                puts
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/ecoportal/api/common/client/error.rb

@@ -0,0 +1,17 @@
+require 'ecoportal/api/common/client/error/checks'
+
+module Ecoportal
+  module API
+    module Common
+      class Client
+        module Error
+          class UnexpectedServerError < StandardError
+            def initialize(msg, code:)
+              super("Code: #{code} -- Error: #{msg}")
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/ecoportal/api/common/client/rate_throttling.rb

@@ -0,0 +1,43 @@
+require 'rate_throttle_client'
+
+module Ecoportal
+  module API
+    module Common
+      class Client
+        module RateThrottling
+          private
+
+          def rate_throttling(strategy: nil, &block)
+            throttle(strategy).call(&block)
+          end
+
+          def throttle(strategy = nil)
+            @throttle ||= {}
+
+            return @throttle[strategy] if @throttle.key?(strategy)
+
+            @throttle[strategy] = throttle_class(strategy).new
+          end
+
+          def throttle_class(strategy = default_throttle_strategy)
+            case strategy
+            when :gradual_decrease
+              RateThrottleClient::ExponentialIncreaseGradualDecrease
+            when :proportional_decrease
+              RateThrottleClient::ExponentialIncreaseProportionalDecrease
+            when :remaining_decrease
+              # requires RateLimit-Remaining in the response
+              RateThrottleClient::ExponentialIncreaseProportionalRemainingDecrease
+            else # :backoff
+              RateThrottleClient::ExponentialBackoff
+            end
+          end
+
+          def default_throttle_strategy
+            :proportional_decrease
+          end
+        end
+      end
+    end
+  end
+end

data/lib/ecoportal/api/common/client/throughput/stats.rb

@@ -0,0 +1,119 @@
+module Ecoportal
+  module API
+    module Common
+      class Client
+        class Throughput
+          class Stats
+            DEFAULT_SD = 1.0
+            DUMMY_DIFF = 0.001
+            CONFIDENCE = {
+              _90:  1645,
+              _95:  1.96,
+              _98:  2.326,
+              _99:  2.576,
+              _995: 2.807
+            }.freeze
+            DEFAULT_CONFI = :_99
+
+            attr_reader :average, :standard_deviation, :count
+            attr_reader :default_margin_error
+
+            def initialize(average, margin: DEFAULT_CONFI)
+              @average              = average.to_f
+              @standard_deviation   = default_sd
+              @count                = 1
+              @default_margin_error = margin || DEFAULT_CONFI
+            end
+
+            def empty?
+              count <= 1
+            end
+
+            def max(value = default_margin_error)
+              confidence_interval(value).max
+            end
+
+            def min(value = default_margin_error)
+              confidence_interval(value).min
+            end
+
+            def record!(value)
+              if count == 1
+                @average = value
+              else
+                @average = ((average * count) + value) / (count + 1)
+              end
+
+              @standard_deviation = new_sd(value)
+              @count += 1
+              average
+            end
+
+            alias_method :<<, :record!
+
+            # @return [Interval]
+            def confidence_interval(value = default_margin_error)
+              me   = margin_error(value)
+              pair = [me * -1, me].map {|err| average + err}
+              (pair.first..pair.last)
+            end
+
+            protected
+
+            def margin_error(value = default_margin_error)
+              to_z(value) * (standard_deviation / Math.sqrt(count))
+            end
+
+            private
+
+            def new_sd(value)
+              pre_sum = (sd_per_item ** 2) * count
+              cur_sum = pre_sum + ((value - average) ** 2)
+              Math.sqrt(cur_sum / count)
+            end
+
+            #         (diff^2) * N
+            #  SD^2 = -------------
+            #             N - 1
+            #
+            #  (diff^2) * N = SD^2 * (N - 1)
+            #
+            #            --------------
+            #  diff = 2 /  SD^2 * (N - 1)
+            #          V   -------------
+            #                 N
+            def sd_per_item
+              base = (sd_for_calcs ** 2) * (count - 1) / count.to_f
+              Math.sqrt(base)
+            end
+
+            def sd_for_calcs
+              return standard_deviation unless empty?
+
+              standard_deviation + dummy_diff
+            end
+
+            def dummy_diff
+              self.class::DUMMY_DIFF
+            end
+
+            def to_z(value)
+              value = default_margin_error unless confidences.key?(value)
+
+              confidences[value]
+            end
+
+            def confidences
+              self.class::CONFIDENCE
+            end
+
+            def default_sd
+              default = self.class::DEFAULT_SD
+              default * average / 10
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/ecoportal/api/common/client/throughput.rb

@@ -0,0 +1,93 @@
+require 'ecoportal/api/common/client/throughput/stats'
+
+module Ecoportal
+  module API
+    module Common
+      class Client
+        class Throughput
+          MIN_THROUGHPUT = 0.2  # records per second
+          MAX_THROUGHPUT = 12
+
+          attr_reader :default_approach
+
+          def initialize(approach = :conservative)
+            @default_approach = approach || :conservative
+          end
+
+          def eta_for(count, approach: default_approach, ratio: self.ratio(approach))
+            to_seconds(count, ratio: ratio)
+          end
+
+          def ratio(approach = default_approach)
+            case approach
+            when :min
+              min_throughput
+            when :last
+              @last_throughput || stats.average
+            when :average
+              stats.average
+            when :optimistic
+              [stats.max, max_throughput].min
+            else # :conservative
+              [stats.min, min_throughput].max
+            end.then do |relation|
+              next relation if relation < max_throughput
+
+              max_throughput
+            end
+          end
+
+          def record!(secs = nil, count: nil)
+            return ratio if secs.to_f.zero? || count.to_i.zero?
+
+            (count / secs.to_f).round(3).tap do |last|
+              stats << @last_throughput = last
+            end
+          end
+
+          # Keeps track of the current processing speed
+          # @note it needs to be called in specific spots
+          def push(value)
+            stats.record!(value)
+          end
+          alias_method :<<, :push
+
+          def around_min_throughput?(margin)
+            rate = ratio
+            return true if rate == min_throughput
+
+            right_under = min_throughput * (1 - margin)
+            return false if rate < right_under
+
+            right_above = min_throughput * (1 + margin)
+            return false if rate > right_above
+
+            true
+          end
+
+          protected
+
+          def stats
+            @stats ||= Stats.new(min_throughput)
+          end
+
+          def to_seconds(count, ratio: min_throughput)
+            return 1 unless count
+
+            (count.ceil / ratio.to_f).ceil
+          end
+
+          private
+
+          def min_throughput
+            self.class::MIN_THROUGHPUT
+          end
+
+          def max_throughput
+            self.class::MAX_THROUGHPUT
+          end
+        end
+      end
+    end
+  end
+end

data/lib/ecoportal/api/common/client/time_out.rb

@@ -0,0 +1,70 @@
+module Ecoportal
+  module API
+    module Common
+      class Client
+        module TimeOut
+          TIMEOUT_MARGIN = 0.15 # adaptative timeout with margin
+          MIN_SIZE       = 5    # requests
+
+          protected
+
+          attr_writer :throughput
+
+          # Keeps track of the current processing speed
+          # @note it needs to be called in specific spots
+          def throughput!(secs = nil, count: nil)
+            throughput.record!(secs, count: count)
+          end
+
+          def throughput
+            @throughput ||= Throughput.new
+          end
+
+          def to_count(value)
+            value ||= 0
+            return min_size if value.positive? && value < min_size
+            return value    if value.positive?
+
+            1
+          end
+
+          private
+
+          def eta_for(count, approach: :conservative)
+            kargs = {}.tap do |params|
+              next params.merge!({ratio: rectified_throughput}) if approach == :conservative
+
+              params.merge!({approach: approach})
+            end
+
+            throughput.eta_for(to_count(count), **kargs)
+          end
+
+          def timeout_for(count, waited: 0, max_wait: nil, approach: :conservative)
+            (waited + eta_for(count, approach: approach)).then do |time|
+              next time unless max_wait
+
+              [max_wait, time].min
+            end
+          end
+
+          def rectified_throughput
+            throughput.ratio * (1 - timeout_margin)
+          end
+
+          def around_min_throughput?
+            throughput.around_min_throughput?(timeout_margin)
+          end
+
+          def timeout_margin
+            self.class::TIMEOUT_MARGIN
+          end
+
+          def min_size
+            self.class::MIN_SIZE
+          end
+        end
+      end
+    end
+  end
+end