ecoportal-api 0.10.5 → 0.10.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 31b8d7b7f90a351a2335c73140458010669a42bf19538f0ec8a65df565881374
-  data.tar.gz: 064e126df498599971d9c3ea999cba7b85eb303d944ea34738f17490c60c435b
+  metadata.gz: f7d949e504087bf41e30ba17c20425eea599e744aae98456ba943f4433dd819e
+  data.tar.gz: c382957de25cce7270caf6fa2615e731fc0aea94f67f43854ef339e97299461d
 SHA512:
-  metadata.gz: b9020a08208cdc1da174e9addb03bb81e5d484fccc461f45e739edaf32a5a998e1b8508f8fdf3663282733352de1e025f2c4486ebda68109d153c4c1c8326d6b
-  data.tar.gz: 2d4fa049b1cb31edcd28d783a50589922ce04560a2cab5e6a409abc7c3b21ffc807aad6a09a592013eb800b4e3c35e2ecc926d0f7c1e12a061df820e0dc1c37b
+  metadata.gz: e0a7e3401365dff83f9b40aa6e11a812540748075dbce2592feac30c656096baa331817ba70b010cce951b5ca22f0abc12832192a144deff41ddc5f1cd1fc8cc
+  data.tar.gz: 6ac12ae619ce68ea87819b020b2745f9de443abc90b3985f68123291e419d18e4286a6a3ba41debbe2586856fea1eab46aac572e835bc38e4a14e9bbd1813b7a

data/CHANGELOG.md

@@ -2,7 +2,7 @@
 
 All notable changes to this project will be documented in this file.
 
-## [0.10.6] - 2024-10-xx
+## [0.10.8] - 2024-11-xx
 
 ### Added
 
@@ -10,6 +10,38 @@ All notable changes to this project will be documented in this file.
 
 ### Fixed
 
+## [0.10.7] - 2024-11-21
+
+### Changed
+
+- Remove unused code
+- `Ecoportal::API::Common::Client.new` the `version` argument doesn't default to `v1` but to `nil`.
+
+### Fixed
+
+- A couple of errors
+- Remove missing api key error log when no client `version` is specified.
+
+## [0.10.6] - 2024-10-28
+
+### Added
+
+- `Ecoportal::API::V1::Job::Status`
+  - **added** methods `started?`, `progressing?` and `progress_increase`
+- `Ecoportal::API::Errors::StartTimeOut` specificity on timeout error.
+
+### Changed
+
+- Refactored `Ecoportal::API::V1::Awaiter` into:
+  - `Awaiter::Timer`: account for last waited, and manage time outs
+  - `Awaiter::StatusFrequency`: next awaiting time
+
+### Fixed
+
+- `Portal::API::V1::Awaiter`
+  - `throughput` based on progress during last wait
+  - this helps to better detect non progressing jobs
+
 ## [0.10.5] - 2024-10-16
 
 ### Fixed

data/lib/ecoportal/api/common/client/elastic_apm_integration.rb

@@ -61,10 +61,10 @@ module Ecoportal
             }.tap do |options|
               # next unless false
 
-              options.merge!({
-                log_level: Logger::DEBUG,
-                log_path:  File.join(__dir__, "elastic_apm.log")
-              })
+              # options.merge!({
+              #   log_level: ::Logger::DEBUG,
+              #   log_path:  File.join('log', "elastic_apm.log")
+              # })
             end
           end
 

data/lib/ecoportal/api/common/client/error/checks.rb

@@ -28,7 +28,7 @@ module Ecoportal
                 msg  = "Received non json body in response "
                 msg << "(#{response.src_body.class}):\n "
                 msg << response.src_body
-                puts
+                puts msg
               end
             end
           end

data/lib/ecoportal/api/common/client/with_retry.rb

@@ -47,7 +47,7 @@ module Ecoportal
               log_unexpected_server_error(response)
 
               msg  = "Got server error (#{response.status}): #{response.body}\n"
-              msg << "Going to retry (##{i} of #{attempts})"
+              msg << "Going to retry (##{i + 1} of #{attempts})"
               log(:debug) { msg }
 
               sleep(delay) if i < attempts

data/lib/ecoportal/api/common/client.rb

@@ -39,7 +39,7 @@ module Ecoportal
         # @param logger [Logger] an object with `Logger` interface to generate logs.
         # @param deep_logging [Boolean] whether or not batch responses should be logged
         # @return [Client] an object that holds the configuration of the api connection.
-        def initialize(api_key:, version: "v1", host: DEFAULT_HOST, logger: nil, deep_logging: false)
+        def initialize(api_key:, version: nil, host: DEFAULT_HOST, logger: nil, deep_logging: false)
           @version      = version
           @api_key      = api_key
           @logger       = logger
@@ -59,7 +59,9 @@ module Ecoportal
           end
 
           return unless @api_key.nil? || @api_key.match(/\A\W*\z/)
+          return unless version
 
+          byebug
           log(:error) { "Api-key missing!" }
         end
 

data/lib/ecoportal/api/common/doc_helpers.rb

@@ -2,7 +2,6 @@ module Ecoportal
   module API
     module Common
       module DocHelpers
-
         def get_body(doc)
           if doc.respond_to?(:as_update)
             doc.as_update

data/lib/ecoportal/api/errors/time_out.rb

@@ -3,6 +3,9 @@ module Ecoportal
     module Errors
       class TimeOut < Errors::Base
       end
+
+      class StartTimeOut < TimeOut
+      end
     end
   end
 end

data/lib/ecoportal/api/v1/job/awaiter/status_frequency.rb

@@ -0,0 +1,49 @@
+module Ecoportal
+  module API
+    class V1
+      class Job
+        class Awaiter
+          module StatusFrequency
+            include Common::Client::TimeOut
+
+            DELAY_STATUS_CHECK = 4
+            MIN_STATUS_CHECK   = 2
+
+            private
+
+            def status_checked?
+              @status_checked || false
+            end
+
+            def status_check_in(pending, timeout_in:)
+              # first check to be faster
+              unless status_checked?
+                @status_checked = true unless pending.zero? # <= increases job status requests!
+                return min_delay_status_check
+              end
+
+              return default_delay_status_check if around_min_throughput?
+
+              eta           = eta_for(pending, approach: :optimistic)
+              check_in_max  = [eta, timeout_in].min * 0.90
+
+              check_in_best = check_in_max / 2.0
+              default_5     = default_delay_status_check * 5
+
+              top_check     = [default_5, check_in_best].min.ceil
+              [top_check, min_delay_status_check].max
+            end
+
+            def default_delay_status_check
+              self.class::DELAY_STATUS_CHECK
+            end
+
+            def min_delay_status_check
+              self.class::MIN_STATUS_CHECK
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/ecoportal/api/v1/job/awaiter/timer.rb

@@ -0,0 +1,158 @@
+module Ecoportal
+  module API
+    class V1
+      class Job
+        class Awaiter
+          TIMER_ARGS = %i[
+            total timeout
+            start last ldelay
+            status lstatus
+          ].freeze
+
+          Timer = Struct.new(*TIMER_ARGS) do
+            self::MAX_START_DELAY = 60
+
+            attr_reader :timestamp
+
+            def initialize(...)
+              @timestamp = Time.now
+              super
+              yield(self) if block_given?
+            end
+
+            alias_method :original_start, :start
+            # `start` time only counts from the moment that
+            # it already started to progress
+            def start
+              return timestamp unless lstatus&.started?
+
+              original_start || timestamp
+            end
+
+            alias_method :original_last, :last
+            def last
+              original_last || start
+            end
+
+            def new(**kargs, &block)
+              self.class.new(**new_kargs.merge(kargs), &block)
+            end
+
+            def job_id
+              status.id
+            end
+
+            def complete?
+              status.complete?(total)
+            end
+
+            def started?
+              status.started?
+            end
+
+            def pending
+              status.pending(total)
+            end
+
+            def progress
+              status.progress
+            end
+
+            def increased
+              status.progress_increase(lstatus)
+            end
+
+            def waited
+              timestamp - start
+            end
+
+            def lwaited
+              timestamp - last
+            end
+
+            def net_waited
+              last_delay = ldelay || 0
+              waited - (last_delay / 2)
+            end
+
+            def time_left
+              (timeout - waited).round(2)
+            end
+
+            def time_left_to_start
+              return max_start_delay if started?
+
+              (max_start_delay - waited).round(2)
+            end
+
+            def timeout_in
+              return time_left if started?
+
+              time_left_to_start
+            end
+
+            # timeout library is evil. So we make poor-man timeout.
+            # https://jvns.ca/blog/2015/11/27/why-rubys-timeout-is-dangerous-and-thread-dot-raise-is-terrifying/
+            def timeout?
+              !time_left.positive?
+            end
+
+            def start_timeout?
+              return false if status.started?
+
+              !time_left_to_start.positive?
+            end
+
+            def on_timeout!(&block)
+              return start_timeout!(&block) if start_timeout?
+              return timeout!(&block)       if timeout?
+
+              false
+            end
+
+            def timeout!
+              return false unless timeout?
+
+              yield(self)
+
+              msg  = "Job '#{job_id}' not complete (size: #{total}).\n"
+              msg << "  Timed out after #{timeout} seconds.\n"
+              msg << "  Current status: #{status}"
+
+              raise API::Errors::TimeOut, msg
+            end
+
+            def start_timeout!
+              return false unless start_timeout?
+
+              yield(self)
+
+              msg  = "Job '#{job_id}' not started (size: #{total}).\n"
+              msg << "  Start timed out after #{max_start_delay} seconds.\n"
+              msg << "  Current status: #{status}"
+
+              raise API::Errors::StartTimeOut, msg
+            end
+
+            protected
+
+            def new_kargs
+              %i[total timeout start].each_with_object({}) do |key, kargs|
+                kargs[key] = send(key)
+              end.tap do |kargs|
+                kargs[:start]   = timestamp
+                kargs[:lstatus] = status
+              end
+            end
+
+            private
+
+            def max_start_delay
+              self.class::MAX_START_DELAY
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/ecoportal/api/v1/job/awaiter.rb

@@ -1,64 +1,77 @@
+require_relative 'awaiter/timer'
+require_relative 'awaiter/status_frequency'
+
 module Ecoportal
   module API
     class V1
       class Job
         class Awaiter
           include Common::Client::TimeOut
-
-          DELAY_STATUS_CHECK = 4
-          MIN_STATUS_CHECK   = 2
+          include StatusFrequency
 
           TIMEOUT_APPROACH   = :min # :conservative # adaptative timeout
           TIMEOUT_FALLBACK   = :min
 
-          attr_reader :job, :job_id, :total
-          attr_accessor :timeout_approach
+          attr_reader :job_api, :job_id, :total
 
-          def initialize(job, job_id:, total:)
-            @job     = job
+          def initialize(job_api, job_id:, total:)
+            @job_api = job_api
             @job_id  = job_id
             @total   = total
-            @checked = false
-            self.timeout_approach = self.class::TIMEOUT_APPROACH
+          end
+
+          attr_writer :timeout_approach
+
+          def timeout_approach
+            @timeout_approach ||= self.class::TIMEOUT_APPROACH
           end
 
           # Allows to preserve the learned throughput
           def new(**kargs)
-            self.class.new(job, **kargs).tap do |out|
-              out.throughput = throughput
+            self.class.new(job_api, **kargs).tap do |out|
+              out.throughput       = throughput
+              out.timeout_approach = timeout_approach
             end
           end
 
           def await_completion! # rubocop:disable Metrics/AbcSize
-            max_timeout = timeout_for(total, approach: timeout_approach)
+            timeout = timeout_for(total, approach: timeout_approach)
+
+            first = 1
+            timer = Timer.new(
+              total:   total,
+              timeout: timeout,
+              ldelay:  first
+            )
 
-            # timeout library is evil. So we make poor-man timeout.
-            # https://jvns.ca/blog/2015/11/27/why-rubys-timeout-is-dangerous-and-thread-dot-raise-is-terrifying/
-            before = Time.now
             delay_status_check = nil
 
             loop do
-              status  = job.status(job_id)
-              done    = status.progress
-              waited  = Time.now - before
+              sleep(first.tap {first = nil}) if first
 
-              adapted = waited
-              adapted = waited - (delay_status_check / 2) if delay_status_check
-              ratio   = throughput!(adapted, count: done)
+              timer = timer.new(
+                status: job_api.status(job_id),
+                ldelay: delay_status_check
+              )
 
-              break status if status.complete?(total)
+              # ratio = throughput!(timer.net_waited, count: timer.progress)
+              ratio = throughput!(timer.lwaited, count: timer.increased)
 
-              pending = status.pending(total)
-              left    = max_timeout - waited
+              break timer.status if timer.complete?
 
-              timeout!(status, timeout: max_timeout) unless left.positive?
+              timer.on_timeout! do
+                @timeout_approach = self.class::TIMEOUT_FALLBACK
+              end
 
-              delay_status_check = status_check_in(pending, timeout_in: left)
+              delay_status_check = status_check_in(
+                timer.pending,
+                timeout_in: timer.timeout_in
+              )
 
               msg  = " ... Awaiting #{delay_status_check} s. -- "
-              msg << " TimeOut: #{left.round(2)} s. "
+              msg << " TimeOut: #{timer.time_left} s. "
               msg << "(job '#{job_id}') "
-              msg << "Done: #{done} (est. #{ratio} rec/s) "
+              msg << "Done: #{timer.progress} (est. #{ratio.round(2)} rec/s) "
               msg << "     \r"
 
               print msg
@@ -67,48 +80,6 @@ module Ecoportal
               sleep(delay_status_check)
             end
           end
-
-          private
-
-          def timeout!(status, timeout:)
-            self.timeout_approach = self.class::TIMEOUT_FALLBACK
-
-            msg  = "Job '#{job_id}' not complete (size: #{total}).\n"
-            msg << "  Timed out after #{timeout} seconds.\n"
-            msg << "  Current status: #{status}"
-
-            raise API::Errors::TimeOut, msg
-          end
-
-          def checked?
-            @checked
-          end
-
-          def status_check_in(pending, timeout_in:)
-            unless checked?
-              @checked = true
-              return min_delay_status_check
-            end
-
-            return default_delay_status_check if around_min_throughput?
-
-            eta           = eta_for(pending, approach: :optimistic)
-            check_in_max  = [eta, timeout_in].min * 0.90
-
-            check_in_best = check_in_max / 2.0
-            default_5     = default_delay_status_check * 5
-
-            top_check     = [default_5, check_in_best].min.ceil
-            [top_check, min_delay_status_check].max
-          end
-
-          def default_delay_status_check
-            self.class::DELAY_STATUS_CHECK
-          end
-
-          def min_delay_status_check
-            self.class::MIN_STATUS_CHECK
-          end
         end
       end
     end

data/lib/ecoportal/api/v1/job/status.rb

@@ -9,7 +9,15 @@ module Ecoportal
             @id       = id
             @complete = complete
             @errored  = errored
-            @progress = progress
+            @progress = progress || 0
+          end
+
+          def started?
+            progress.positive?
+          end
+
+          def progressing?(prev)
+            progress_increase(prev).positive?
           end
 
           def complete?(total = nil)
@@ -25,6 +33,12 @@ module Ecoportal
             total - progress
           end
 
+          def progress_increase(prev = 0)
+            prev   = prev.progress if prev.is_a?(self.class)
+            prev ||= 0
+            progress - prev
+          end
+
           def errored?
             @errored
           end

data/lib/ecoportal/api/version.rb

@@ -1,5 +1,5 @@
 module Ecoportal
   module API
-    VERSION = '0.10.5'.freeze
+    VERSION = '0.10.7'.freeze
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ecoportal-api
 version: !ruby/object:Gem::Version
-  version: 0.10.5
+  version: 0.10.7
 platform: ruby
 authors:
 - Tapio Saarinen
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2024-10-15 00:00:00.000000000 Z
+date: 2024-11-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: pry
@@ -258,6 +258,8 @@ files:
 - lib/ecoportal/api/v1.rb
 - lib/ecoportal/api/v1/job.rb
 - lib/ecoportal/api/v1/job/awaiter.rb
+- lib/ecoportal/api/v1/job/awaiter/status_frequency.rb
+- lib/ecoportal/api/v1/job/awaiter/timer.rb
 - lib/ecoportal/api/v1/job/status.rb
 - lib/ecoportal/api/v1/people.rb
 - lib/ecoportal/api/v1/person.rb
@@ -287,7 +289,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.5.18
+rubygems_version: 3.5.23
 signing_key:
 specification_version: 4
 summary: A collection of helpers for interacting with the ecoPortal MS's various APIs