aspera-cli 4.24.1 → 4.25.0.pre

data/lib/aspera/rest.rb

@@ -50,84 +50,95 @@ module Aspera
     end
   end
 
+  class EntityNotFound < Error
+  end
+
   # a simple class to make HTTP calls, equivalent to rest-client
   # rest call errors are raised as exception RestCallError
   # and error are analyzed in RestErrorAnalyzer
   class Rest
-    # flag for array parameters prefixed with []
-    ARRAY_PARAMS = '[]'
-
-    private_constant :ARRAY_PARAMS
-
-    # error message when entity not found (TODO: use specific exception)
-    ENTITY_NOT_FOUND = 'No such'
-
     MIME_JSON = 'application/json'
     MIME_WWW = 'application/x-www-form-urlencoded'
     MIME_TEXT = 'text/plain'
 
-    # Content-Type that are JSON
-    JSON_DECODE = [MIME_JSON, 'application/vnd.api+json', 'application/x-javascript'].freeze
-
-    UNAVAILABLE_CODES = ['503']
+    # Special query parameter: max number of items for list command
+    MAX_ITEMS = 'max'
+    # Special query parameter: max number of pages for list command
+    MAX_PAGES = 'pmax'
 
     class << self
       # @return [String] Basic auth token
       def basic_authorization(user, pass); return "Basic #{Base64.strict_encode64("#{user}:#{pass}")}"; end
 
-      # Build a parameter list prefixed with "[]"
-      # @param values [Array] list of values
-      def array_params(values)
-        return [ARRAY_PARAMS].concat(values)
-      end
-
-      def array_params?(values)
-        return values.first.eql?(ARRAY_PARAMS)
+      # Indicate that the given Hash query uses php style for array parameters
+      # @param query [Hash] A key can have Array value and result will use PHP format: a[]=1&a[]=2
+      def php_style(query)
+        Aspera.assert_type(query, Hash){'query'}
+        query[:x_array_php_style] = true
+        query
       end
 
       # Build URI from URL and parameters and check it is http or https
-      # encode array [] parameters
-      # @param query [Hash,Array]
-      def build_uri(url, query = nil)
+      # Check iof php style is specified
+      # @param url   [String]            The URL without query.
+      # @param query [Hash,Array,String] The query.
+      def build_uri(url, query)
         uri = URI.parse(url)
-        Aspera.assert(%w[http https].include?(uri.scheme)){"REST endpoint shall be http/s not #{uri.scheme}"}
+        Aspera.assert_values(uri.scheme, %w[http https]){'URI scheme'}
         return uri if query.nil? || query.respond_to?(:empty?) && query.empty?
         Log.dump(:query, query)
-        query_array = []
-        case query
-        when Hash
-          query.each do |k, v|
-            case v
-            when Array
-              # support array for query parameter, there is no standard. Either p[]=1&p[]=2, or p=1&p=2
-              suffix = array_params?(v) ? v.shift : ''
-              v.each do |e|
-                query_array.push(["#{k}#{suffix}", e])
-              end
-            else
-              query_array.push([k, v])
+        uri.query =
+          case query
+          when String
+            query
+          when Hash
+            URI.encode_www_form(h_to_query_array(query))
+          when Array
+            Aspera.assert(query.all?{ |i| i.is_a?(Array) && i.length.eql?(2)}){'Query must be array of arrays of 2 elements'}
+            URI.encode_www_form(query)
+          else Aspera.error_unexpected_value(query.class){'query type'}
+          end.gsub('%5B%5D=', '[]=')
+        # [] is allowed in url parameters
+        uri
+      end
+
+      # @param query [Hash] HTTP query as hash
+      def h_to_query_array(query)
+        Aspera.assert_type(query, Hash)
+        # Support array for query parameter, there is no standard.
+        # Either p[]=1&p[]=2, or p=1&p=2
+        suffix = query.delete(:x_array_php_style) ? '[]' : nil
+        query.each_with_object([]) do |(k, v), query_array|
+          case v
+          when Array
+            v.each do |e|
+              query_array.push(["#{k}#{suffix}", e])
             end
+          else
+            query_array.push([k, v])
           end
-        when Array
-          Aspera.assert(query.all?{ |i| i.is_a?(Array) && i.length.eql?(2)}){'Query must be array of arrays or 2 elements'}
-          query_array = query
-        else
-          raise "Query must be Hash or Array, not #{query.class}"
         end
-        # [] is allowed in url parameters
-        uri.query = URI.encode_www_form(query_array).gsub('%5B%5D=', '[]=')
-        return uri
       end
 
       # Decode query string as Hash
-      # Does not support arrays in query string, no standard, e.g. PHP's way is p[]=1&p[]=2
+      # if parameter is only once, then it's a scalar
+      # if a parameter is several, then it's array
+      # if parameter has [] then it's an array, and [] is removed
+      # Support arrays in query string, e.g. PHP's way is p[]=1&p[]=2
       # @param query [String] query string as in URI.query
       # @return [Hash] decoded query
       def query_to_h(query)
-        URI.decode_www_form(query).each_with_object({}) do |pair, h|
-          key = pair.first
-          raise "Array not supported in query string: #{key}" if key.include?('[]') || h.key?(key)
-          h[key] = pair.last
+        URI.decode_www_form(query).each_with_object({}) do |(key, value), h|
+          if key.end_with?('[]')
+            key = key[..-3]
+            h[key] = [] unless h.key?(key)
+          end
+          if h.key?(key)
+            h[key] = [h[key]] if !h[key].is_a?(Array)
+            h[key].push(value)
+          else
+            h[key] = value
+          end
         end
       end
 
@@ -135,13 +146,14 @@ module Aspera
       # @param base_url [String] base url of HTTP/S session
       # @return [Net::HTTP] a started HTTP session
       def start_http_session(base_url)
-        uri = build_uri(base_url)
-        # this honors http_proxy env var
+        uri = URI.parse(base_url)
+        Aspera.assert_values(uri.scheme, %w[http https]){'URI scheme'}
+        # This honors http_proxy env var
         http_session = Net::HTTP.new(uri.host, uri.port)
         http_session.use_ssl = uri.scheme.eql?('https')
-        # set http options in callback, such as timeout and cert. verification
+        # Set http options in callback, such as timeout and cert. verification
         RestParameters.instance.session_cb&.call(http_session)
-        # manually start session for keep alive (if supported by server, else, session is closed every time)
+        # Manually start session for keep alive (if supported by server, else, session is closed every time)
         http_session.start
         return http_session
       end
@@ -174,19 +186,31 @@ module Aspera
         return result
       end
 
-      # Parse a header string as returned by HTTP
-      # @param header [String] header string, e.g. "application/json; charset=utf-8"
-      # @return [Hash] parsed header with type and parameters
-      #   {type: 'application/json', parameters: {charset: 'utf-8'}}
+      # Parses an HTTP Content-Type header string into its media type and parameters
+      # according to RFC 9110 and RFC 6838.
+      # TODO: use gem: content_type
+      #
+      # @param header [String] The Content-Type header string, e.g., "application/json; charset=utf-8"
+      # @return [Hash] A hash with :type and :parameters keys.
+      #   Example:
+      #     {
+      #       type: "application/json",
+      #       parameters: {
+      #         charset: "utf-8",
+      #         version: "1.0"
+      #       }
+      #     }
       def parse_header(header)
-        type, *params = header.split(/;\s*/)
-        parameters = params.map do |param|
-          one = param.split(/=\s*/)
-          one[0] = one[0].to_sym
-          one[1] = one[1].gsub(/\A"|"\z/, '')
-          one
+        parts = header.split(';').map(&:strip)
+        media_type = parts.shift.downcase
+        parameters = parts.filter_map do |param|
+          key, value = param.split('=', 2)
+          next unless key && value
+          key = key.strip.downcase.to_sym
+          value = value.strip.gsub(/\A"|"\z/, '')
+          [key, value]
         end.to_h
-        {type: type.downcase, parameters: parameters}
+        {type: media_type, parameters: parameters}
       end
     end
 
@@ -200,8 +224,13 @@ module Aspera
 
     public
 
-    attr_reader :base_url
+    # All original constructor parameters
     attr_reader :auth_params
+
+    # The root URL for the API
+    attr_reader :base_url
+
+    # Base common headers of API
     attr_reader :headers
 
     # @return creation parameters
@@ -271,14 +300,16 @@ module Aspera
     end
 
     # HTTP/S REST call
-    # @param operation [String] HTTP operation (GET, POST, PUT, DELETE)
-    # @param subpath [String] subpath of REST API
-    # @param query [Hash] URL parameters
-    # @param content_type [String,nil] Type of body parameters (one of MIME_*) and serialization, else use headers
-    # @param body [Hash, String] body parameters
-    # @param headers [Hash] additional headers (override Content-Type)
-    # @param save_to_file (filepath)
-    # @param return_error (bool)
+    # @param operation    [String] HTTP operation (GET, POST, PUT, DELETE)
+    # @param subpath      [String] subpath of REST API
+    # @param query        [Hash] URL parameters
+    # @param content_type [String, nil] Type of body parameters (one of MIME_*) and serialization, else use headers
+    # @param body       [Hash, String] body parameters
+    # @param headers      [Hash] additional headers (override Content-Type)
+    # @param save_to_file [String, nil](filepath)
+    # @param exception    [Bool] `true`, error raise exception
+    # @param ret [:data, :resp, :both] Tell to return only data, only http response, or both
+    # @return [Object, Array] only data, only http response, or both
     def call(
       operation:,
       subpath: nil,
@@ -287,13 +318,18 @@ module Aspera
       body: nil,
       headers: nil,
       save_to_file: nil,
-      return_error: false
+      exception: true,
+      ret: :data
     )
       subpath = subpath.to_s if subpath.is_a?(Symbol)
       subpath = '' if subpath.nil?
       Log.log.debug{"call #{operation} [#{subpath}]".red.bold.bg_green}
-      Log.dump(:body, body)
+      Log.dump(:body, body, level: :trace1)
+      Log.dump(:query, query, level: :trace1)
+      Log.dump(:headers, headers, level: :trace1)
       Aspera.assert_type(subpath, String)
+      # We must have a way to check return code
+      Aspera.assert(exception || !ret.eql?(:data))
       if headers.nil?
         headers = @headers.clone
       else
@@ -317,7 +353,8 @@ module Aspera
         end
       else Aspera.error_unexpected_value(@auth_params[:type])
       end
-      result = {http: nil}
+      result_http = nil
+      result_data = nil
       # start a block to be able to retry the actual HTTP request in case of OAuth token expiration
       begin
         # TODO: shall we percent encode subpath (spaces) test with access key delete with space in id
@@ -360,14 +397,14 @@ module Aspera
         file_saved = false
         # make http request (pipelined)
         http_session.request(req) do |response|
-          result[:http] = response
-          result_mime = self.class.parse_header(result[:http]['Content-Type'] || MIME_TEXT)[:type]
-          Log.log.debug{"response: code=#{result[:http].code}, mime=#{result_mime}, mime2= #{response['Content-Type']}"}
+          result_http = response
+          result_mime = self.class.parse_header(result_http['Content-Type'] || MIME_TEXT)[:type]
+          Log.log.debug{"response: code=#{result_http.code}, mime=#{result_mime}, mime2= #{response['Content-Type']}"}
           # JSON data needs to be parsed, in case it contains an error code
           if !save_to_file.nil? &&
-              result[:http].code.to_s.start_with?('2') &&
+              result_http.code.to_s.start_with?('2') &&
               !JSON_DECODE.include?(result_mime)
-            total_size = result[:http]['Content-Length']&.to_i
+            total_size = result_http['Content-Length']&.to_i
             Log.log.debug('before write file')
             target_file = save_to_file
             # override user's path to path in header
@@ -385,7 +422,7 @@ module Aspera
             FileUtils.mkdir_p(File.dirname(target_file_tmp))
             limiter = TimerLimiter.new(0.5)
             File.open(target_file_tmp, 'wb') do |file|
-              result[:http].read_body do |fragment|
+              result_http.read_body do |fragment|
                 file.write(fragment)
                 written_size += fragment.length
                 RestParameters.instance.progress_bar&.event(:transfer, session_id: session_id, info: written_size) if limiter.trigger?
@@ -398,32 +435,32 @@ module Aspera
             file_saved = true
           end
         end
-        Log.log.debug{"result: code=#{result[:http].code} mime=#{result_mime}"}
+        Log.log.debug{"result: code=#{result_http.code} mime=#{result_mime}"}
         # sometimes there is a UTF8 char (e.g. (c) ), TODO : related to mime type encoding ?
-        # result[:http].body.force_encoding('UTF-8') if result[:http].body.is_a?(String)
-        # Log.log.debug{"result: body=#{result[:http].body}"}
+        # result_http.body.force_encoding('UTF-8') if result_http.body.is_a?(String)
+        # Log.log.debug{"result: body=#{result_http.body}"}
         case result_mime
         when *JSON_DECODE
-          result[:data] = JSON.parse(result[:http].body) rescue result[:http].body
-          Log.dump(:result_data, result[:data])
+          result_data = JSON.parse(result_http.body) rescue result_http.body
+          Log.dump(:result_data, result_data)
         else # when MIME_TEXT
-          result[:data] = result[:http].body
+          result_data = result_http.body
         end
-        RestErrorAnalyzer.instance.raise_on_error(req, result)
+        RestErrorAnalyzer.instance.raise_on_error(req, result_data, result_http)
         unless file_saved || save_to_file.nil?
           FileUtils.mkdir_p(File.dirname(save_to_file))
-          File.write(save_to_file, result[:http].body, binmode: true)
+          File.write(save_to_file, result_http.body, binmode: true)
         end
       rescue RestCallError => e
         do_retry = false
         # AoC have some timeout , like Connect to platform.bss.asperasoft.com:443 ...
         do_retry ||= true if e.response.body.include?('failed: connect timed out') && RestParameters.instance.retry_on_timeout
         # AoC sometimes not available
-        do_retry ||= true if RestParameters.instance.retry_on_unavailable && UNAVAILABLE_CODES.include?(result[:http].code.to_s)
+        do_retry ||= true if RestParameters.instance.retry_on_unavailable && UNAVAILABLE_CODES.include?(result_http.code.to_s)
         # possibility to retry anything if it fails
         do_retry ||= true if RestParameters.instance.retry_on_error
         # not authorized: oauth token expired
-        if @not_auth_codes.include?(result[:http].code.to_s) && @auth_params[:type].eql?(:oauth2)
+        if @not_auth_codes.include?(result_http.code.to_s) && @auth_params[:type].eql?(:oauth2)
           begin
             # try to use refresh token
             req['Authorization'] = oauth.authorization(refresh: true)
@@ -462,15 +499,21 @@ module Aspera
             body: body,
             content_type: content_type,
             save_to_file: save_to_file,
-            return_error: return_error,
-            headers: headers
+            exception: exception,
+            headers: headers,
+            ret: ret
           )
         end
         # raise exception if could not retry and not return error in result
-        raise e unless return_error
+        raise e if exception
       end
-      Log.log.debug{"result=http:#{result[:http]}, data:#{result[:data].class}"}
-      return result
+      Log.log.debug{"result=http:#{result_http}, data:#{result_data.class}"}
+      return case ret
+             when :data then result_data
+             when :resp then result_http
+             when :both then [result_data, result_http]
+             else Aspera.error_unexpected_value(ret){'Type of result for REST'}
+             end
     end
 
     #
@@ -478,31 +521,36 @@ module Aspera
     # If specific elements are needed, then use the full `call` method
     #
 
-    def create(subpath, params)
-      return call(operation: 'POST', subpath: subpath, headers: {'Accept' => MIME_JSON}, body: params, content_type: MIME_JSON)[:data]
+    # Create: `POST`
+    def create(subpath, params, **kwargs)
+      return call(operation: 'POST', subpath: subpath, headers: {'Accept' => MIME_JSON}, body: params, content_type: MIME_JSON, **kwargs)
     end
 
-    def read(subpath, query = nil)
-      return call(operation: 'GET', subpath: subpath, headers: {'Accept' => MIME_JSON}, query: query)[:data]
+    # Read: `GET`
+    def read(subpath, query = nil, **kwargs)
+      return call(operation: 'GET', subpath: subpath, headers: {'Accept' => MIME_JSON}, query: query, **kwargs)
     end
 
-    def update(subpath, params)
-      return call(operation: 'PUT', subpath: subpath, headers: {'Accept' => MIME_JSON}, body: params, content_type: MIME_JSON)[:data]
+    # Update: `PUT`
+    def update(subpath, params, **kwargs)
+      return call(operation: 'PUT', subpath: subpath, headers: {'Accept' => MIME_JSON}, body: params, content_type: MIME_JSON, **kwargs)
     end
 
-    def delete(subpath, params = nil)
-      return call(operation: 'DELETE', subpath: subpath, headers: {'Accept' => MIME_JSON}, query: params)[:data]
+    # Delete: `DELETE`
+    def delete(subpath, params = nil, **kwargs)
+      return call(operation: 'DELETE', subpath: subpath, headers: {'Accept' => MIME_JSON}, query: params, **kwargs)
     end
 
-    def cancel(subpath)
-      return call(operation: 'CANCEL', subpath: subpath, headers: {'Accept' => MIME_JSON})[:data]
+    # Cancel: `CANCEL`
+    def cancel(subpath, **kwargs)
+      return call(operation: 'CANCEL', subpath: subpath, headers: {'Accept' => MIME_JSON}, **kwargs)
     end
 
     # Query entity by general search (read with parameter `q`)
     # TODO: not generic enough ? move somewhere ? inheritance ?
-    # @param subpath path of entity in API
-    # @param search_name name of searched entity
-    # @param query additional search query parameters
+    # @param subpath     [String] Path of entity in API
+    # @param search_name [String] Name of searched entity
+    # @param query       [Hash]   Additional search query parameters
     # @returns [Hash] A single entity matching the search, or an exception if not found or multiple found
     def lookup_by_name(subpath, search_name, query: nil)
       query = {} if query.nil?
@@ -513,17 +561,24 @@ module Aspera
       Aspera.assert_type(matching_items, Array)
       case matching_items.length
       when 1 then return matching_items.first
-      when 0 then raise %Q{#{ENTITY_NOT_FOUND} #{subpath}: "#{search_name}"}
+      when 0 then raise EntityNotFound, %Q{No such #{subpath}: "#{search_name}"}
       else
         # multiple case insensitive partial matches, try case insensitive full match
         # (anyway AoC does not allow creation of 2 entities with same case insensitive name)
         name_matches = matching_items.select{ |i| i['name'].casecmp?(search_name)}
         case name_matches.length
         when 1 then return name_matches.first
-        when 0 then raise %Q(#{subpath}: multiple case insensitive partial match for: "#{search_name}": #{matching_items.map{ |i| i['name']}} but no case insensitive full match. Please be more specific or give exact name.)
+        when 0 then raise %Q(#{subpath}: Multiple case insensitive partial match for: "#{search_name}": #{matching_items.map{ |i| i['name']}} but no case insensitive full match. Please be more specific or give exact name.)
         else raise "Two entities cannot have the same case insensitive name: #{name_matches.map{ |i| i['name']}}"
         end
       end
     end
+
+    # Content-Type that are JSON
+    JSON_DECODE = [MIME_JSON, 'application/vnd.api+json', 'application/x-javascript'].freeze
+
+    UNAVAILABLE_CODES = ['503']
+
+    private_constant :JSON_DECODE, :UNAVAILABLE_CODES
   end
 end

data/lib/aspera/rest_error_analyzer.rb

@@ -27,13 +27,13 @@ module Aspera
     # Use this method to analyze a EST result and raise an exception
     # Analyzes REST call response and raises a RestCallError exception
     # if HTTP result code is not 2XX
-    def raise_on_error(req, res)
-      Log.log.debug{"raise_on_error #{req.method} #{req.path} #{res[:http].code}"}
+    def raise_on_error(req, data, http)
+      Log.log.debug{"raise_on_error #{req.method} #{req.path} #{http.code}"}
       call_context = {
         messages: [],
         request:  req,
-        response: res[:http],
-        data:     res[:data]
+        response: http,
+        data:     data
       }
       # multiple error messages can be found
       # analyze errors from provided handlers

data/lib/aspera/ssh.rb

@@ -11,19 +11,21 @@ module Aspera
     class Error < Aspera::Error
     end
     class << self
+      # HACK: disable some key type
       def disable_ed25519_keys
         Log.log.debug('Disabling SSH ed25519 user keys')
         old_verbose = $VERBOSE
         $VERBOSE = nil
         Net::SSH::Authentication::Session.class_eval do
           define_method(:default_keys) do
-            %w[~/.ssh/id_dsa ~/.ssh/id_rsa ~/.ssh2/id_dsa ~/.ssh2/id_rsa].freeze
+            %w[.ssh .ssh2].product(%w[rsa dsa ecdsa]).map{"~/#{_1}/id_#{_2}"}.freeze
           end
           private(:default_keys)
         end rescue nil
         $VERBOSE = old_verbose
       end
 
+      # HACK: disable some algorithms
       def disable_ecd_sha2_algorithms
         Log.log.debug('Disabling SSH ecdsa')
         Net::SSH::Transport::Algorithms::ALGORITHMS.each_value{ |a| a.reject!{ |a| a =~ /^ecd(sa|h)-sha2/}}
@@ -38,6 +40,7 @@ module Aspera
       Aspera.assert_type(host, String)
       Aspera.assert_type(username, String)
       Aspera.assert_type(ssh_options, Hash)
+      ssh_options[:use_agent] = false unless ssh_options.key?(:use_agent)
       @host = host
       @username = username
       @ssh_options = ssh_options
@@ -62,7 +65,7 @@ module Aspera
             exit_code = data.read_long
             next if exit_code.zero?
             error_message = "#{cmd}: exit #{exit_code}, #{error.join.chomp}"
-            raise Error, error_message if  exception
+            raise Error, error_message if exception
             # Happens when windows user hasn't logged in and created home account.
             error_message += "\nHint: home not created in Windows?" if data.include?('Could not chdir to home directory')
             Log.log.debug(error_message)
@@ -81,6 +84,6 @@ module Aspera
   end
 end
 
-# HACK: deactivate ed25519 and ecdsa private keys from SSH identities, as it usually causes problems
-Aspera::Ssh.disable_ed25519_keys if ENV.fetch('ASCLI_ENABLE_ED25519', 'false').eql?('false')
+# Deactivate ed25519 and ecdsa private keys from SSH identities, as it usually causes problems
+Aspera::Ssh.disable_ed25519_keys if Gem::Specification.find_all_by_name('ed25519').none? || ENV.fetch('ASCLI_ENABLE_ED25519', 'true').eql?('false')
 Aspera::Ssh.disable_ecd_sha2_algorithms if defined?(JRUBY_VERSION) && ENV.fetch('ASCLI_ENABLE_ECDSHA2', 'false').eql?('false')

data/lib/aspera/ssl.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+require 'openssl'
+require 'aspera/assert'
+require 'aspera/log'
+
+module Aspera
+  # Give possibility to globally override SSL options
+  module SSL
+    @extra_options = 0
+    class << self
+      @extra_options = OpenSSL::SSL::SSLContext::DEFAULT_PARAMS[:options]
+      attr_reader :extra_options
+
+      def option_list=(v)
+        Aspera.assert_type(v, Array){'ssl_options'}
+        v.each do |opt|
+          Aspera.assert_type(opt, String, Integer){'Expected String or Integer in ssl_options'}
+          case opt
+          when Integer
+            @extra_options = opt
+          when String
+            name = "OP_#{opt.start_with?('-') ? opt[1..] : opt}".upcase
+            raise Cli::BadArgument, "Unknown ssl_option: #{name}, use one of: #{OpenSSL::SSL.constants.grep(/^OP_/).map{ |c| c.to_s.sub(/^OP_/, '')}.join(', ')}" if !OpenSSL::SSL.const_defined?(name)
+            if opt.start_with?('-')
+              @extra_options &= ~OpenSSL::SSL.const_get(name)
+            else
+              @extra_options |= OpenSSL::SSL.const_get(name)
+            end
+          end
+        end
+      end
+    end
+    def set_params(params = {})
+      super(params)
+      self.options = Aspera::SSL.extra_options unless Aspera::SSL.extra_options.nil?
+      self
+    end
+  end
+end
+OpenSSL::SSL::SSLContext.prepend(Aspera::SSL)