etna 0.1.47 → 0.1.49

data/lib/commands.rb

@@ -261,8 +261,8 @@ class EtnaApp
             model_attributes_mask: model_attribute_pairs(project_name),
             record_names: 'all',
             model_filters: {},
-            metis_client: metis_client,
-            magma_client: magma_client,
+            metis_client: metis_client(logger: logger),
+            magma_client: magma_client(logger: logger),
             logger: logger,
             project_name: project_name,
             model_name: 'project', filesystem: filesystem,

data/lib/etna/client.rb

@@ -4,10 +4,17 @@ require 'rack/utils'
 
 module Etna
   class Client
-    def initialize(host, token, routes_available: true, ignore_ssl: false)
+
+    def initialize(host, token, routes_available: true, ignore_ssl: false, max_retries: 10, backoff_time: 15, logger: nil)
       @host = host.sub(%r!/$!, '')
       @token = token
       @ignore_ssl = ignore_ssl
+      @max_retries = max_retries
+      @backoff_time = backoff_time
+
+      default_logger = ::Etna::Logger.new('/dev/stdout', 0, 1048576)
+      default_logger.level = ::Logger::WARN
+      @logger = logger || default_logger
 
       if routes_available
         set_routes
@@ -55,7 +62,7 @@ module Etna
       uri = request_uri(endpoint)
       multipart = Net::HTTP::Post::Multipart.new uri.request_uri, content
       multipart.add_field('Authorization', "Etna #{@token}")
-      request(uri, multipart, &block)
+      retrier.retry_request(uri, multipart, &block)
     end
 
     def post(endpoint, params = {}, &block)
@@ -80,9 +87,12 @@ module Etna
 
     private
 
+    def retrier
+      @retrier ||= Retrier.new(ignore_ssl: @ignore_ssl, max_retries: @max_retries, backoff_time: @backoff_time, logger: @logger)
+    end
+
     def set_routes
       response = options('/')
-      status_check!(response)
       @routes = JSON.parse(response.body, symbolize_names: true)
     end
 
@@ -120,7 +130,7 @@ module Etna
       uri = request_uri(endpoint)
       req = type.new(uri.request_uri, request_headers)
       req.body = params.to_json
-      request(uri, req, &block)
+      retrier.retry_request(uri, req, &block)
     end
 
     def query_request(type, endpoint, params = {}, &block)
@@ -132,7 +142,7 @@ module Etna
         uri.query = URI.encode_www_form(params)
       end
       req = type.new(uri.request_uri, request_headers)
-      request(uri, req, &block)
+      retrier.retry_request(uri, req, &block)
     end
 
     def request_uri(endpoint)
@@ -140,6 +150,8 @@ module Etna
     end
 
     def request_headers
+      refresh_token
+
       {
           'Content-Type' => 'application/json',
           'Accept' => 'application/json, text/*',
@@ -149,46 +161,234 @@ module Etna
       )
     end
 
-    def status_check!(response)
-      status = response.code.to_i
-      if status >= 400
-        msg = response.content_type == 'application/json' ?
-            json_error(response.body) :
-            response.body
-        raise Etna::Error.new(msg, status)
-      end
+    def refresh_token
+      @token = TokenRefresher.new(@host, @token, @logger).active_token
     end
 
-    def json_error(body)
-      msg = JSON.parse(body, symbolize_names: true)
-      if (msg.has_key?(:errors) && msg[:errors].is_a?(Array))
-        return JSON.generate(msg[:errors])
-      elsif msg.has_key?(:error)
-        return JSON.generate(msg[:error])
+    class TokenRefresher
+      def initialize(host, token, logger)
+        @token = token
+        @host = host
+        @logger = logger
+      end
+
+      def active_token
+        token_will_expire? ?
+          refresh_token :
+          @token
+      end
+
+      private
+
+      def token_expired?
+        # Has the token already expired?
+        token_will_expire?(0)
+      end
+
+      def token_will_expire?(offset=3000)
+        return false if @token.nil?
+
+        # Will the user's token expire in the given amount of time?
+        payload = @token.split('.')[1]
+        return false if payload.nil?
+
+        epoch_seconds = JSON.parse(Base64.urlsafe_decode64(payload))["exp"]
+
+        return false if epoch_seconds.nil?
+
+        expiration = DateTime.strptime(epoch_seconds.to_s, "%s").to_time
+        expiration <= DateTime.now.new_offset.to_time + offset
+      end
+
+      def refresh_token
+        @logger.debug("Requesting a refreshed token.")
+        uri = refresh_uri
+        req = Net::HTTP::Post.new(uri.request_uri, request_headers)
+        retrier.retry_request(uri, req).body
+      end
+
+      def refresh_uri
+        URI("#{janus_host}#{refresh_endpoint}")
+      end
+
+      def janus_host
+        @host.gsub(/(metis|magma|timur|polyphemus|janus|gnomon)/, "janus")
+      end
+
+      def refresh_endpoint
+        "/api/tokens/generate"
+      end
+
+      def request_headers
+        {
+          'Content-Type' => 'application/json',
+          'Accept' => 'application/json, text/*',
+          'Authorization' => "Etna #{@token}"
+        }
+      end
+
+      def retrier
+        @retrier ||= Retrier.new(max_retries: 5, backoff_time: 10, logger: @logger)
       end
     end
 
-    def request(uri, data)
-      if block_given?
-        verify_mode = @ignore_ssl ?
-          OpenSSL::SSL::VERIFY_NONE :
-          OpenSSL::SSL::VERIFY_PEER
-        Net::HTTP.start(uri.host, uri.port, use_ssl: true, verify_mode: verify_mode, read_timeout: 300) do |http|
-          http.request(data) do |response|
+    class Retrier
+      # Ideally the retry code would be centralized with metis_client ...
+      #   unsure what would be the best approach to do that, at this moment.
+      def initialize(ignore_ssl: false, max_retries: 10, backoff_time: 15, logger:)
+        @max_retries = max_retries
+        @ignore_ssl = ignore_ssl
+        @backoff_time = backoff_time
+        @logger = logger
+      end
+
+      def retry_request(uri, req, retries: 0, &block)
+        retries += 1
+
+        begin
+          @logger.debug("\rWaiting for #{uri.host} restart"+"."*retries+"\x1b[0K")
+
+          sleep @backoff_time * retries
+        end if retries > 1
+
+        if retries < @max_retries
+          begin
+            if block_given?
+              request(uri, req) do |block_response|
+                if net_exceptions.include?(block_response.class)
+                  @logger.debug("Received #{block_response.class.name}, retrying")
+                  retry_request(uri, req, retries: retries, &block)
+                elsif block_response.is_a?(OpenSSL::SSL::SSLError)
+                  @logger.debug("SSL error, retrying")
+                  retry_request(uri, req, retries: retries, &block)
+                else
+                  status_check!(block_response)
+                  yield block_response
+                end
+              end
+            else
+              response = request(uri, req)
+            end
+          rescue OpenSSL::SSL::SSLError => e
+            if e.message =~ /write client hello/
+              @logger.debug("SSL error, retrying")
+              return retry_request(uri, req, retries: retries)
+            end
+            raise e
+          rescue *net_exceptions => e
+            @logger.debug("Received #{e.class.name}, retrying")
+            return retry_request(uri, req, retries: retries)
+          end
+
+          begin
+            retry_codes = ['503', '502', '504', '408']
+            if retry_codes.include?(response.code)
+              @logger.debug("Received response with code #{response.code}, retrying")
+              return retry_request(uri, req, retries: retries)
+            elsif response.code == '500' && response.body.start_with?("Puma caught")
+              @logger.debug("Received 500 Puma error #{response.body.split("\n").first}, retrying")
+              return retry_request(uri, req, retries: retries)
+            end
+
             status_check!(response)
-            yield response
+            return response
+          end unless block_given?
+        else
+          raise ::Etna::Error, "Could not contact server, giving up"
+        end
+      end
+
+      private
+
+      def request(uri, data)
+        if block_given?
+          verify_mode = @ignore_ssl ?
+            OpenSSL::SSL::VERIFY_NONE :
+            OpenSSL::SSL::VERIFY_PEER
+          Net::HTTP.start(uri.host, uri.port, use_ssl: true, verify_mode: verify_mode, read_timeout: 300) do |http|
+            http.request(data) do |response|
+              api_error_check!(response)
+              yield response
+            end
+          end
+        else
+          verify_mode = @ignore_ssl ?
+            OpenSSL::SSL::VERIFY_NONE :
+            OpenSSL::SSL::VERIFY_PEER
+          Net::HTTP.start(uri.host, uri.port, use_ssl: true, verify_mode: verify_mode, read_timeout: 300) do |http|
+            response = http.request(data)
+            api_error_check!(response)
+            return response
           end
         end
-      else
-        verify_mode = @ignore_ssl ?
-          OpenSSL::SSL::VERIFY_NONE :
-          OpenSSL::SSL::VERIFY_PEER
-        Net::HTTP.start(uri.host, uri.port, use_ssl: true, verify_mode: verify_mode, read_timeout: 300) do |http|
-          response = http.request(data)
-          status_check!(response)
-          return response
+      end
+
+      def api_error_check!(response)
+        status = response.code.to_i
+        raise_status_error(response) if 400 <= status && status < 500
+      end
+
+      def status_check!(response)
+        status = response.code.to_i
+        raise_status_error(response) if status >= 400
+      end
+
+      def raise_status_error(response)
+        msg = response.content_type == 'application/json' ?
+          json_error(response.body) :
+          response.body
+        raise Etna::Error.new(msg, response.code.to_i)
+      end
+
+      def json_error(body)
+        msg = JSON.parse(body, symbolize_names: true)
+        if (msg.has_key?(:errors) && msg[:errors].is_a?(Array))
+          return JSON.generate(msg[:errors])
+        elsif msg.has_key?(:error)
+          return JSON.generate(msg[:error])
         end
       end
+
+      def net_exceptions
+        retry_exceptions = [
+          Errno::ECONNREFUSED,
+          Errno::ECONNRESET,
+          Errno::ENETRESET,
+          Errno::EPIPE,
+          Errno::ECONNABORTED,
+          Errno::EHOSTDOWN,
+          Errno::EHOSTUNREACH,
+          Errno::EINVAL,
+          Errno::ETIMEDOUT,
+          Net::ReadTimeout,
+          Net::HTTPFatalError,
+          Net::HTTPBadResponse,
+          Net::HTTPHeaderSyntaxError,
+          Net::ProtocolError,
+          Net::HTTPRequestTimeOut,
+          Net::HTTPGatewayTimeOut,
+          Net::HTTPBadRequest,
+          Net::HTTPBadGateway,
+          Net::HTTPError,
+          Net::HTTPInternalServerError,
+          Net::HTTPRetriableError,
+          Net::HTTPServerError,
+          Net::HTTPServiceUnavailable,
+          Net::HTTPUnprocessableEntity,
+          Net::OpenTimeout,
+          IOError,
+          EOFError,
+          Timeout::Error
+        ]
+
+        begin
+          retry_exceptions << Net::HTTPRequestTimeout
+          retry_exceptions << Net::HTTPGatewayTimeout
+          retry_exceptions << Net::WriteTimeout
+        end if RUBY_VERSION > "2.5.8"
+
+        retry_exceptions
+      end
     end
   end
 end

data/lib/etna/clients/base_client.rb

@@ -6,7 +6,7 @@ module Etna
   module Clients
     class BaseClient
       attr_reader :host, :token, :ignore_ssl
-      def initialize(host:, token:, ignore_ssl: false)
+      def initialize(host:, token:, ignore_ssl: false, logger: nil)
         raise "#{self.class.name} client configuration is missing host." unless host
 
         @token = token
@@ -16,7 +16,8 @@ module Etna
           host,
           token,
           routes_available: false,
-          ignore_ssl: ignore_ssl)
+          ignore_ssl: ignore_ssl,
+          logger: logger)
         @host = host
         @ignore_ssl = ignore_ssl
       end

data/lib/etna/clients/magma/workflows/walk_model_tree_workflow.rb

@@ -94,7 +94,6 @@ module Etna
               collections = []
               links = []
               attributes = []
-
               model = response.models.model(model_name)
 
               template.attributes.attribute_keys.each do |attr_name|

data/lib/etna/clients/metis/client.rb

@@ -10,10 +10,10 @@ module Etna
   module Clients
     class Metis < Etna::Clients::BaseClient
 
-      def initialize(host:, token:, ignore_ssl: false)
+      def initialize(host:, token:, ignore_ssl: false, logger: nil)
         raise 'Metis client configuration is missing host.' unless host
         raise 'Metis client configuration is missing token.' unless token
-        @etna_client = ::Etna::Client.new(host, token, ignore_ssl: ignore_ssl)
+        @etna_client = ::Etna::Client.new(host, token, ignore_ssl: ignore_ssl, logger: logger)
 
         @token = token
       end

data/lib/etna/clients/metis/workflows/sync_metis_data_workflow.rb

@@ -81,7 +81,7 @@ module Etna
 
                 upload_timings << [chunk.length, Time.now.to_f]
                 upload_amount += chunk.length
-                remaining -= chunk.length
+                remaining -= chunk.length unless remaining.nil?
 
                 if upload_timings.length > 150
                   s, _ = upload_timings.shift

data/lib/etna/formatting.rb

@@ -3,6 +3,8 @@ module Etna
     SIZE_UNITS = %W(B KiB MiB GiB TiB).freeze
 
     def self.as_size(number)
+      return "unknown" if number.nil?
+
       if number.to_i < 1024
         exponent = 0
       else

data/lib/etna/route.rb

@@ -1,5 +1,6 @@
 require 'digest'
 require 'date'
+require 'net/http'
 require "addressable/uri"
 require_relative "./censor"
 require_relative './instrumentation'
@@ -46,7 +47,7 @@ module Etna
       if params
         PARAM_TYPES.reduce(route) do |path,pat|
           path.gsub(pat) do
-            params[$1.to_sym].split('/').map { |c| block_given? ? yield(c) : Addressable::URI.normalized_encode(c) }.join('/')
+            params[$1.to_sym].split('/').map { |c| block_given? ? yield(c) : Etna::Route.encode_path_component(c) }.join('/')
           end
         end
       else
@@ -54,6 +55,12 @@ module Etna
       end
     end
 
+    def self.encode_path_component(path_component)
+      form_encoded = URI.encode_www_form_component(path_component)
+      # Because of issues #1005, #1006, #1197
+      form_encoded.gsub('+', '%20')
+    end
+
     def path(params=nil, &block)
       self.class.path(@route, params, &block)
     end

data/lib/etna/spec/auth.rb

@@ -31,9 +31,17 @@ module Etna::Spec
     def below_admin_roles
       [:editor, :viewer, :guest]
     end
-    
+
     def below_editor_roles
       [:viewer, :guest]
     end
+
+    def stub_janus_refresh
+      stub_request(:post, /\/api\/tokens\/generate/)
+      .to_return({
+        status: 200,
+        body: ""
+      })
+    end
   end
 end

data/lib/helpers.rb

@@ -43,17 +43,19 @@ module WithEtnaClients
     env_token
   end
 
-  def magma_client
+  def magma_client(logger: nil)
     @magma_client ||= Etna::Clients::Magma.new(
         token: token,
         ignore_ssl: EtnaApp.instance.config(:ignore_ssl),
+        logger: logger,
         **EtnaApp.instance.config(:magma, environment) || {})
   end
 
-  def metis_client
+  def metis_client(logger: nil)
     @metis_client ||= Etna::Clients::Metis.new(
         token: token,
         ignore_ssl: EtnaApp.instance.config(:ignore_ssl),
+        logger: logger,
         **EtnaApp.instance.config(:metis, environment) || {})
   end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: etna
 version: !ruby/object:Gem::Version
-  version: 0.1.47
+  version: 0.1.49
 platform: ruby
 authors:
 - Saurabh Asthana
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2023-01-25 00:00:00.000000000 Z
+date: 2023-02-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rack