aspera-cli 4.24.2 → 4.25.0.pre2

data/lib/aspera/rest.rb

@@ -50,13 +50,13 @@ module Aspera
     end
   end
 
+  class EntityNotFound < Error
+  end
+
   # a simple class to make HTTP calls, equivalent to rest-client
   # rest call errors are raised as exception RestCallError
   # and error are analyzed in RestErrorAnalyzer
   class Rest
-    # Error message when entity not found (TODO: use specific exception)
-    ENTITY_NOT_FOUND = 'No such'
-
     MIME_JSON = 'application/json'
     MIME_WWW = 'application/x-www-form-urlencoded'
     MIME_TEXT = 'text/plain'
@@ -71,7 +71,7 @@ module Aspera
       def basic_authorization(user, pass); return "Basic #{Base64.strict_encode64("#{user}:#{pass}")}"; end
 
       # Indicate that the given Hash query uses php style for array parameters
-      # a[]=1&a[]=2
+      # @param query [Hash] A key can have Array value and result will use PHP format: a[]=1&a[]=2
       def php_style(query)
         Aspera.assert_type(query, Hash){'query'}
         query[:x_array_php_style] = true
@@ -300,14 +300,16 @@ module Aspera
     end
 
     # HTTP/S REST call
-    # @param operation [String] HTTP operation (GET, POST, PUT, DELETE)
-    # @param subpath [String] subpath of REST API
-    # @param query [Hash] URL parameters
-    # @param content_type [String,nil] Type of body parameters (one of MIME_*) and serialization, else use headers
-    # @param body [Hash, String] body parameters
-    # @param headers [Hash] additional headers (override Content-Type)
-    # @param save_to_file (filepath)
-    # @param exception (bool) true, error raise exception
+    # @param operation    [String] HTTP operation (GET, POST, PUT, DELETE)
+    # @param subpath      [String] subpath of REST API
+    # @param query        [Hash] URL parameters
+    # @param content_type [String, nil] Type of body parameters (one of MIME_*) and serialization, else use headers
+    # @param body       [Hash, String] body parameters
+    # @param headers      [Hash] additional headers (override Content-Type)
+    # @param save_to_file [String, nil](filepath)
+    # @param exception    [Bool] `true`, error raise exception
+    # @param ret [:data, :resp, :both] Tell to return only data, only http response, or both
+    # @return [Object, Array] only data, only http response, or both
     def call(
       operation:,
       subpath: nil,
@@ -316,7 +318,8 @@ module Aspera
       body: nil,
       headers: nil,
       save_to_file: nil,
-      exception: true
+      exception: true,
+      ret: :data
     )
       subpath = subpath.to_s if subpath.is_a?(Symbol)
       subpath = '' if subpath.nil?
@@ -325,6 +328,8 @@ module Aspera
       Log.dump(:query, query, level: :trace1)
       Log.dump(:headers, headers, level: :trace1)
       Aspera.assert_type(subpath, String)
+      # We must have a way to check return code
+      Aspera.assert(exception || !ret.eql?(:data))
       if headers.nil?
         headers = @headers.clone
       else
@@ -348,7 +353,8 @@ module Aspera
         end
       else Aspera.error_unexpected_value(@auth_params[:type])
       end
-      result = {http: nil}
+      result_http = nil
+      result_data = nil
       # start a block to be able to retry the actual HTTP request in case of OAuth token expiration
       begin
         # TODO: shall we percent encode subpath (spaces) test with access key delete with space in id
@@ -391,14 +397,14 @@ module Aspera
         file_saved = false
         # make http request (pipelined)
         http_session.request(req) do |response|
-          result[:http] = response
-          result_mime = self.class.parse_header(result[:http]['Content-Type'] || MIME_TEXT)[:type]
-          Log.log.debug{"response: code=#{result[:http].code}, mime=#{result_mime}, mime2= #{response['Content-Type']}"}
+          result_http = response
+          result_mime = self.class.parse_header(result_http['Content-Type'] || MIME_TEXT)[:type]
+          Log.log.debug{"response: code=#{result_http.code}, mime=#{result_mime}, mime2= #{response['Content-Type']}"}
           # JSON data needs to be parsed, in case it contains an error code
           if !save_to_file.nil? &&
-              result[:http].code.to_s.start_with?('2') &&
+              result_http.code.to_s.start_with?('2') &&
               !JSON_DECODE.include?(result_mime)
-            total_size = result[:http]['Content-Length']&.to_i
+            total_size = result_http['Content-Length']&.to_i
             Log.log.debug('before write file')
             target_file = save_to_file
             # override user's path to path in header
@@ -416,7 +422,7 @@ module Aspera
             FileUtils.mkdir_p(File.dirname(target_file_tmp))
             limiter = TimerLimiter.new(0.5)
             File.open(target_file_tmp, 'wb') do |file|
-              result[:http].read_body do |fragment|
+              result_http.read_body do |fragment|
                 file.write(fragment)
                 written_size += fragment.length
                 RestParameters.instance.progress_bar&.event(:transfer, session_id: session_id, info: written_size) if limiter.trigger?
@@ -429,32 +435,32 @@ module Aspera
             file_saved = true
           end
         end
-        Log.log.debug{"result: code=#{result[:http].code} mime=#{result_mime}"}
+        Log.log.debug{"result: code=#{result_http.code} mime=#{result_mime}"}
         # sometimes there is a UTF8 char (e.g. (c) ), TODO : related to mime type encoding ?
-        # result[:http].body.force_encoding('UTF-8') if result[:http].body.is_a?(String)
-        # Log.log.debug{"result: body=#{result[:http].body}"}
+        # result_http.body.force_encoding('UTF-8') if result_http.body.is_a?(String)
+        # Log.log.debug{"result: body=#{result_http.body}"}
         case result_mime
         when *JSON_DECODE
-          result[:data] = JSON.parse(result[:http].body) rescue result[:http].body
-          Log.dump(:result_data, result[:data])
+          result_data = JSON.parse(result_http.body) rescue result_http.body
+          Log.dump(:result_data, result_data)
         else # when MIME_TEXT
-          result[:data] = result[:http].body
+          result_data = result_http.body
         end
-        RestErrorAnalyzer.instance.raise_on_error(req, result)
+        RestErrorAnalyzer.instance.raise_on_error(req, result_data, result_http)
         unless file_saved || save_to_file.nil?
           FileUtils.mkdir_p(File.dirname(save_to_file))
-          File.write(save_to_file, result[:http].body, binmode: true)
+          File.write(save_to_file, result_http.body, binmode: true)
         end
       rescue RestCallError => e
         do_retry = false
         # AoC have some timeout , like Connect to platform.bss.asperasoft.com:443 ...
         do_retry ||= true if e.response.body.include?('failed: connect timed out') && RestParameters.instance.retry_on_timeout
         # AoC sometimes not available
-        do_retry ||= true if RestParameters.instance.retry_on_unavailable && UNAVAILABLE_CODES.include?(result[:http].code.to_s)
+        do_retry ||= true if RestParameters.instance.retry_on_unavailable && UNAVAILABLE_CODES.include?(result_http.code.to_s)
         # possibility to retry anything if it fails
         do_retry ||= true if RestParameters.instance.retry_on_error
         # not authorized: oauth token expired
-        if @not_auth_codes.include?(result[:http].code.to_s) && @auth_params[:type].eql?(:oauth2)
+        if @not_auth_codes.include?(result_http.code.to_s) && @auth_params[:type].eql?(:oauth2)
           begin
             # try to use refresh token
             req['Authorization'] = oauth.authorization(refresh: true)
@@ -494,14 +500,20 @@ module Aspera
             content_type: content_type,
             save_to_file: save_to_file,
             exception: exception,
-            headers: headers
+            headers: headers,
+            ret: ret
           )
         end
         # raise exception if could not retry and not return error in result
         raise e if exception
       end
-      Log.log.debug{"result=http:#{result[:http]}, data:#{result[:data].class}"}
-      return result
+      Log.log.debug{"result=http:#{result_http}, data:#{result_data.class}"}
+      return case ret
+             when :data then result_data
+             when :resp then result_http
+             when :both then [result_data, result_http]
+             else Aspera.error_unexpected_value(ret){'Type of result for REST'}
+             end
     end
 
     #
@@ -511,27 +523,27 @@ module Aspera
 
     # Create: `POST`
     def create(subpath, params, **kwargs)
-      return call(operation: 'POST', subpath: subpath, headers: {'Accept' => MIME_JSON}, body: params, content_type: MIME_JSON, **kwargs)[:data]
+      return call(operation: 'POST', subpath: subpath, headers: {'Accept' => MIME_JSON}, body: params, content_type: MIME_JSON, **kwargs)
     end
 
     # Read: `GET`
     def read(subpath, query = nil, **kwargs)
-      return call(operation: 'GET', subpath: subpath, headers: {'Accept' => MIME_JSON}, query: query, **kwargs)[:data]
+      return call(operation: 'GET', subpath: subpath, headers: {'Accept' => MIME_JSON}, query: query, **kwargs)
     end
 
     # Update: `PUT`
     def update(subpath, params, **kwargs)
-      return call(operation: 'PUT', subpath: subpath, headers: {'Accept' => MIME_JSON}, body: params, content_type: MIME_JSON, **kwargs)[:data]
+      return call(operation: 'PUT', subpath: subpath, headers: {'Accept' => MIME_JSON}, body: params, content_type: MIME_JSON, **kwargs)
     end
 
     # Delete: `DELETE`
     def delete(subpath, params = nil, **kwargs)
-      return call(operation: 'DELETE', subpath: subpath, headers: {'Accept' => MIME_JSON}, query: params, **kwargs)[:data]
+      return call(operation: 'DELETE', subpath: subpath, headers: {'Accept' => MIME_JSON}, query: params, **kwargs)
     end
 
     # Cancel: `CANCEL`
     def cancel(subpath, **kwargs)
-      return call(operation: 'CANCEL', subpath: subpath, headers: {'Accept' => MIME_JSON}, **kwargs)[:data]
+      return call(operation: 'CANCEL', subpath: subpath, headers: {'Accept' => MIME_JSON}, **kwargs)
     end
 
     # Query entity by general search (read with parameter `q`)
@@ -549,7 +561,7 @@ module Aspera
       Aspera.assert_type(matching_items, Array)
       case matching_items.length
       when 1 then return matching_items.first
-      when 0 then raise %Q{#{ENTITY_NOT_FOUND} #{subpath}: "#{search_name}"}
+      when 0 then raise EntityNotFound, %Q{No such #{subpath}: "#{search_name}"}
       else
         # multiple case insensitive partial matches, try case insensitive full match
         # (anyway AoC does not allow creation of 2 entities with same case insensitive name)

data/lib/aspera/rest_error_analyzer.rb

@@ -27,13 +27,13 @@ module Aspera
     # Use this method to analyze a EST result and raise an exception
     # Analyzes REST call response and raises a RestCallError exception
     # if HTTP result code is not 2XX
-    def raise_on_error(req, res)
-      Log.log.debug{"raise_on_error #{req.method} #{req.path} #{res[:http].code}"}
+    def raise_on_error(req, data, http)
+      Log.log.debug{"raise_on_error #{req.method} #{req.path} #{http.code}"}
       call_context = {
         messages: [],
         request:  req,
-        response: res[:http],
-        data:     res[:data]
+        response: http,
+        data:     data
       }
       # multiple error messages can be found
       # analyze errors from provided handlers

data/lib/aspera/ssh.rb

@@ -11,6 +11,7 @@ module Aspera
     class Error < Aspera::Error
     end
     class << self
+      # HACK: disable some key type
       def disable_ed25519_keys
         Log.log.debug('Disabling SSH ed25519 user keys')
         old_verbose = $VERBOSE
@@ -24,6 +25,7 @@ module Aspera
         $VERBOSE = old_verbose
       end
 
+      # HACK: disable some algorithms
       def disable_ecd_sha2_algorithms
         Log.log.debug('Disabling SSH ecdsa')
         Net::SSH::Transport::Algorithms::ALGORITHMS.each_value{ |a| a.reject!{ |a| a =~ /^ecd(sa|h)-sha2/}}
@@ -38,6 +40,7 @@ module Aspera
       Aspera.assert_type(host, String)
       Aspera.assert_type(username, String)
       Aspera.assert_type(ssh_options, Hash)
+      ssh_options[:use_agent] = false unless ssh_options.key?(:use_agent)
       @host = host
       @username = username
       @ssh_options = ssh_options
@@ -62,7 +65,7 @@ module Aspera
             exit_code = data.read_long
             next if exit_code.zero?
             error_message = "#{cmd}: exit #{exit_code}, #{error.join.chomp}"
-            raise Error, error_message if  exception
+            raise Error, error_message if exception
             # Happens when windows user hasn't logged in and created home account.
             error_message += "\nHint: home not created in Windows?" if data.include?('Could not chdir to home directory')
             Log.log.debug(error_message)
@@ -82,5 +85,5 @@ module Aspera
 end
 
 # Deactivate ed25519 and ecdsa private keys from SSH identities, as it usually causes problems
-Aspera::Ssh.disable_ed25519_keys if Gem::Specification.find_all_by_name('ed25519').none?
+Aspera::Ssh.disable_ed25519_keys if Gem::Specification.find_all_by_name('ed25519').none? || ENV.fetch('ASCLI_ENABLE_ED25519', 'true').eql?('false')
 Aspera::Ssh.disable_ecd_sha2_algorithms if defined?(JRUBY_VERSION) && ENV.fetch('ASCLI_ENABLE_ECDSHA2', 'false').eql?('false')

data/lib/aspera/ssl.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+require 'openssl'
+require 'aspera/assert'
+require 'aspera/log'
+
+module Aspera
+  # Give possibility to globally override SSL options
+  module SSL
+    @extra_options = 0
+    class << self
+      @extra_options = OpenSSL::SSL::SSLContext::DEFAULT_PARAMS[:options]
+      attr_reader :extra_options
+
+      def option_list=(v)
+        Aspera.assert_type(v, Array){'ssl_options'}
+        v.each do |opt|
+          Aspera.assert_type(opt, String, Integer){'Expected String or Integer in ssl_options'}
+          case opt
+          when Integer
+            @extra_options = opt
+          when String
+            name = "OP_#{opt.start_with?('-') ? opt[1..] : opt}".upcase
+            raise Cli::BadArgument, "Unknown ssl_option: #{name}, use one of: #{OpenSSL::SSL.constants.grep(/^OP_/).map{ |c| c.to_s.sub(/^OP_/, '')}.join(', ')}" if !OpenSSL::SSL.const_defined?(name)
+            if opt.start_with?('-')
+              @extra_options &= ~OpenSSL::SSL.const_get(name)
+            else
+              @extra_options |= OpenSSL::SSL.const_get(name)
+            end
+          end
+        end
+      end
+    end
+    def set_params(params = {})
+      super(params)
+      self.options = Aspera::SSL.extra_options unless Aspera::SSL.extra_options.nil?
+      self
+    end
+  end
+end
+OpenSSL::SSL::SSLContext.prepend(Aspera::SSL)