aspera-cli 4.24.1 → 4.24.2

data/lib/aspera/oauth/base.rb

@@ -13,6 +13,7 @@ module Aspera
     # OAuth 2.0 Authorization Framework: https://tools.ietf.org/html/rfc6749
     # Bearer Token Usage: https://tools.ietf.org/html/rfc6750
     class Base
+      Aspera.require_method!(:create_token)
       # @param **             Parameters for REST
       # @param client_id      [String, nil]
       # @param client_secret  [String, nil]
@@ -31,8 +32,7 @@ module Aspera
         cache_ids: nil,
         **rest_params
       )
-        Aspera.assert(respond_to?(:create_token), 'create_token method must be defined', type: InternalError)
-        # this is the OAuth API
+        # This is the OAuth API
         @api = Rest.new(**rest_params)
         @scope = nil
         @token_cache_id = nil
@@ -43,6 +43,7 @@ module Aspera
         @use_query = use_query
         @base_cache_ids = cache_ids.nil? ? [] : cache_ids.clone
         Aspera.assert_type(@base_cache_ids, Array)
+        # TODO: this shall be done in class, using cache_ids
         @base_cache_ids.push(@api.auth_params[:username]) if @api.auth_params.key?(:username)
         @base_cache_ids.compact!
         @base_cache_ids.freeze
@@ -56,16 +57,20 @@ module Aspera
         @token_cache_id = Factory.cache_id(@api.base_url, self.class, @base_cache_ids, @scope)
       end
 
-      attr_reader :scope, :api, :path_token
+      attr_reader :scope, :api, :path_token, :client_id
 
       # helper method to create token as per RFC
       def create_token_call(creation_params)
         Log.log.debug{'Generating a new token'.bg_green}
-        payload = {content_type: Rest::MIME_WWW}
-        if @use_query
-          payload[:query] = creation_params
+        payload = if @use_query
+          {
+            query: creation_params
+          }
         else
-          payload[:body] = creation_params
+          {
+            content_type: Rest::MIME_WWW,
+            body:         creation_params
+          }
         end
         return @api.call(
           operation: 'POST',
@@ -75,12 +80,13 @@ module Aspera
         )
       end
 
-      # @return Hash with optional general parameters
+      # @param add_secret [Boolean] Add secret in default call parameters
+      # @return [Hash] Optional general parameters
       def optional_scope_client_id(add_secret: false)
         call_params = {}
         call_params[:scope] = @scope unless @scope.nil?
         call_params[:client_id] = @client_id unless @client_id.nil?
-        call_params[:client_secret] = @client_secret if add_secret && !@client_id.nil?
+        call_params[:client_secret] = @client_secret unless !add_secret || @client_id.nil? || @client_secret.nil?
         return call_params
       end
 
@@ -106,6 +112,7 @@ module Aspera
           # `direct` agent is equipped with refresh code
           # an API was already called, but failed, we need to regenerate or refresh
           if refresh || token_info[:expired]
+            Log.log.trace1{"refresh: #{refresh} expired: #{token_info[:expired]}"}
             refresh_token = nil
             if token_data.key?('refresh_token') && !token_data['refresh_token'].eql?('not_supported')
               # save possible refresh token, before deleting the cache
@@ -118,7 +125,7 @@ module Aspera
             if !refresh_token.nil?
               Log.log.info{"refresh=[#{refresh_token}]".bg_green}
               # NOTE: AoC admin token has no refresh, and lives by default 1800secs
-              resp = create_token_call(optional_scope_client_id.merge(grant_type: 'refresh_token', refresh_token: refresh_token))
+              resp = create_token_call(optional_scope_client_id(add_secret: true).merge(grant_type: 'refresh_token', refresh_token: refresh_token))
               if resp[:http].code.start_with?('2')
                 # save only if success
                 json_data = resp[:http].body

data/lib/aspera/oauth/factory.rb

@@ -63,10 +63,10 @@ module Aspera
         @decoders = []
         # default parameters, others can be added by handlers
         @parameters = {
-          # tokens older than 30 minutes will be discarded from cache
-          token_cache_expiry_sec:     1800,
-          # tokens valid for less than this duration will be regenerated
-          token_expiration_guard_sec: 120
+          # tokens older than this duration in sec. will be discarded from cache
+          token_cache_max_age:     1800,
+          # tokens valid for less than this duration in sec. will be regenerated
+          token_refresh_threshold: 120
         }
       end
 
@@ -77,7 +77,7 @@ module Aspera
       def persist_mgr=(manager)
         @persist = manager
         # cleanup expired tokens
-        @persist.garbage_collect(PERSIST_CATEGORY_TOKEN, @parameters[:token_cache_expiry_sec])
+        @persist.garbage_collect(PERSIST_CATEGORY_TOKEN, @parameters[:token_cache_max_age])
       end
 
       def persist_mgr
@@ -108,7 +108,7 @@ module Aspera
         end
       end
 
-      # get token information from cache
+      # Get token information from cache
       # @param id [String] identifier of token
       # @return [Hash] token internal information , including Date object for `expiration_date`
       def get_token_info(id)
@@ -118,7 +118,6 @@ module Aspera
         Aspera.assert_type(token_data, Hash)
         decoded_token = decode_token(token_data[TOKEN_FIELD])
         info = {data: token_data}
-        Log.dump(:decoded_token, decoded_token)
         if decoded_token.is_a?(Hash)
           info[:decoded] = decoded_token
           # TODO: move date decoding to token decoder ?
@@ -129,9 +128,10 @@ module Aspera
           unless expiration_date.nil?
             info[:expiration] = expiration_date
             info[:ttl_sec] = expiration_date - Time.now
-            info[:expired] = info[:ttl_sec] < @parameters[:token_expiration_guard_sec]
+            info[:expired] = info[:ttl_sec] < @parameters[:token_refresh_threshold]
           end
         end
+        Log.dump(:token_info, info)
         return info
       end
 

data/lib/aspera/oauth/web.rb

@@ -9,7 +9,7 @@ module Aspera
     # Authentication using Web browser
     class Web < Base
       class << self
-        attr_accessor :additionnal_info
+        attr_accessor :additional_info
       end
       # @param redirect_uri    url to receive the code after auth (to be exchanged for token)
       # @param path_authorize  path to login page on web app
@@ -37,7 +37,7 @@ module Aspera
         # here, we need a human to authorize on a web page
         Log.log.info{"login_page_url=#{login_page_url}".bg_red.gray}
         # start a web server to receive request code
-        web_server = WebAuth.new(@redirect_uri, self.class.additionnal_info)
+        web_server = WebAuth.new(@redirect_uri, self.class.additional_info)
         # start browser on login page
         Environment.instance.open_uri(login_page_url)
         # wait for code in request

data/lib/aspera/products/connect.rb

@@ -48,11 +48,12 @@ module Aspera
         end
       end
 
+      # Base URL for CDN of Connect
       def cdn_api
         Rest.new(base_url: CDN_BASE_URL)
       end
 
-      # retrieve structure from cloud (CDN) with all versions available
+      # Retrieve structure from cloud (CDN) with all versions available
       def versions
         if @connect_versions.nil?
           javascript = cdn_api.call(operation: 'GET', subpath: VERSION_INFO_FILE)
@@ -74,10 +75,10 @@ module Aspera
         @connect_versions = nil
       end
 
-      VERSION_INFO_FILE = 'connectversions.js' # cspell: disable-line
       CDN_BASE_URL = 'https://d3gcli72yxqn2z.cloudfront.net/connect'
+      VERSION_INFO_FILE = 'connectversions.js' # cspell: disable-line
 
-      private_constant :VERSION_INFO_FILE, :CDN_BASE_URL
+      private_constant :CDN_BASE_URL, :VERSION_INFO_FILE
     end
   end
 end

data/lib/aspera/products/desktop.rb

@@ -8,6 +8,7 @@ module Aspera
     class Desktop
       APP_NAME = 'IBM Aspera for Desktop'
       APP_IDENTIFIER = 'com.ibm.software.aspera.desktop'
+      LOG_FILENAME = 'ibm-aspera-desktop.log'
       class << self
         # standard folder locations
         def locations
@@ -20,10 +21,6 @@ module Aspera
           else []
           end.map{ |i| i.merge({expected: APP_NAME})}
         end
-
-        def log_file
-          File.join(Dir.home, 'Library', 'Logs', APP_IDENTIFIER, 'ibm-aspera-desktop.log')
-        end
       end
     end
   end

data/lib/aspera/products/other.rb

@@ -54,8 +54,14 @@ module Aspera
       }]
       end
       class << self
+        # Find installed products and provide paths for it.
+        # @param scan_locations [Array] Array of Hash with keys: expected, app_root, sub_bin, ascp_path, name, version
+        # @return [Array] of products found, with filled missing fields
+        # @raise Exception if no installed product found
         def find(scan_locations)
-          scan_locations.select do |item|
+          product_names = []
+          found = scan_locations.select do |item|
+            product_names.push(item[:expected]) unless product_names.include?(item[:expected])
             # skip if not main folder
             Log.log.trace1{"Checking #{item[:app_root]}"}
             next false unless Dir.exist?(item[:app_root])
@@ -75,6 +81,8 @@ module Aspera
             end
             true # select this version
           end
+          raise "Product: #{product_names.join(', ')} not found, please install." if found.empty?
+          found
         end
       end
     end

data/lib/aspera/products/transferd.rb

@@ -30,7 +30,6 @@ module Aspera
         # @return the path to folder where SDK is installed
         def sdk_directory
           Aspera.assert(!@sdk_dir.nil?){'SDK path was not initialized'}
-          FileUtils.mkdir_p(@sdk_dir)
           @sdk_dir
         end
 

data/lib/aspera/rest.rb

@@ -54,80 +54,91 @@ module Aspera
   # rest call errors are raised as exception RestCallError
   # and error are analyzed in RestErrorAnalyzer
   class Rest
-    # flag for array parameters prefixed with []
-    ARRAY_PARAMS = '[]'
-
-    private_constant :ARRAY_PARAMS
-
-    # error message when entity not found (TODO: use specific exception)
+    # Error message when entity not found (TODO: use specific exception)
     ENTITY_NOT_FOUND = 'No such'
 
     MIME_JSON = 'application/json'
     MIME_WWW = 'application/x-www-form-urlencoded'
     MIME_TEXT = 'text/plain'
 
-    # Content-Type that are JSON
-    JSON_DECODE = [MIME_JSON, 'application/vnd.api+json', 'application/x-javascript'].freeze
-
-    UNAVAILABLE_CODES = ['503']
+    # Special query parameter: max number of items for list command
+    MAX_ITEMS = 'max'
+    # Special query parameter: max number of pages for list command
+    MAX_PAGES = 'pmax'
 
     class << self
       # @return [String] Basic auth token
       def basic_authorization(user, pass); return "Basic #{Base64.strict_encode64("#{user}:#{pass}")}"; end
 
-      # Build a parameter list prefixed with "[]"
-      # @param values [Array] list of values
-      def array_params(values)
-        return [ARRAY_PARAMS].concat(values)
-      end
-
-      def array_params?(values)
-        return values.first.eql?(ARRAY_PARAMS)
+      # Indicate that the given Hash query uses php style for array parameters
+      # a[]=1&a[]=2
+      def php_style(query)
+        Aspera.assert_type(query, Hash){'query'}
+        query[:x_array_php_style] = true
+        query
       end
 
       # Build URI from URL and parameters and check it is http or https
-      # encode array [] parameters
-      # @param query [Hash,Array]
-      def build_uri(url, query = nil)
+      # Check iof php style is specified
+      # @param url   [String]            The URL without query.
+      # @param query [Hash,Array,String] The query.
+      def build_uri(url, query)
         uri = URI.parse(url)
-        Aspera.assert(%w[http https].include?(uri.scheme)){"REST endpoint shall be http/s not #{uri.scheme}"}
+        Aspera.assert_values(uri.scheme, %w[http https]){'URI scheme'}
         return uri if query.nil? || query.respond_to?(:empty?) && query.empty?
         Log.dump(:query, query)
-        query_array = []
-        case query
-        when Hash
-          query.each do |k, v|
-            case v
-            when Array
-              # support array for query parameter, there is no standard. Either p[]=1&p[]=2, or p=1&p=2
-              suffix = array_params?(v) ? v.shift : ''
-              v.each do |e|
-                query_array.push(["#{k}#{suffix}", e])
-              end
-            else
-              query_array.push([k, v])
+        uri.query =
+          case query
+          when String
+            query
+          when Hash
+            URI.encode_www_form(h_to_query_array(query))
+          when Array
+            Aspera.assert(query.all?{ |i| i.is_a?(Array) && i.length.eql?(2)}){'Query must be array of arrays of 2 elements'}
+            URI.encode_www_form(query)
+          else Aspera.error_unexpected_value(query.class){'query type'}
+          end.gsub('%5B%5D=', '[]=')
+        # [] is allowed in url parameters
+        uri
+      end
+
+      # @param query [Hash] HTTP query as hash
+      def h_to_query_array(query)
+        Aspera.assert_type(query, Hash)
+        # Support array for query parameter, there is no standard.
+        # Either p[]=1&p[]=2, or p=1&p=2
+        suffix = query.delete(:x_array_php_style) ? '[]' : nil
+        query.each_with_object([]) do |(k, v), query_array|
+          case v
+          when Array
+            v.each do |e|
+              query_array.push(["#{k}#{suffix}", e])
             end
+          else
+            query_array.push([k, v])
           end
-        when Array
-          Aspera.assert(query.all?{ |i| i.is_a?(Array) && i.length.eql?(2)}){'Query must be array of arrays or 2 elements'}
-          query_array = query
-        else
-          raise "Query must be Hash or Array, not #{query.class}"
         end
-        # [] is allowed in url parameters
-        uri.query = URI.encode_www_form(query_array).gsub('%5B%5D=', '[]=')
-        return uri
       end
 
       # Decode query string as Hash
-      # Does not support arrays in query string, no standard, e.g. PHP's way is p[]=1&p[]=2
+      # if parameter is only once, then it's a scalar
+      # if a parameter is several, then it's array
+      # if parameter has [] then it's an array, and [] is removed
+      # Support arrays in query string, e.g. PHP's way is p[]=1&p[]=2
       # @param query [String] query string as in URI.query
       # @return [Hash] decoded query
       def query_to_h(query)
-        URI.decode_www_form(query).each_with_object({}) do |pair, h|
-          key = pair.first
-          raise "Array not supported in query string: #{key}" if key.include?('[]') || h.key?(key)
-          h[key] = pair.last
+        URI.decode_www_form(query).each_with_object({}) do |(key, value), h|
+          if key.end_with?('[]')
+            key = key[..-3]
+            h[key] = [] unless h.key?(key)
+          end
+          if h.key?(key)
+            h[key] = [h[key]] if !h[key].is_a?(Array)
+            h[key].push(value)
+          else
+            h[key] = value
+          end
         end
       end
 
@@ -135,13 +146,14 @@ module Aspera
       # @param base_url [String] base url of HTTP/S session
       # @return [Net::HTTP] a started HTTP session
       def start_http_session(base_url)
-        uri = build_uri(base_url)
-        # this honors http_proxy env var
+        uri = URI.parse(base_url)
+        Aspera.assert_values(uri.scheme, %w[http https]){'URI scheme'}
+        # This honors http_proxy env var
         http_session = Net::HTTP.new(uri.host, uri.port)
         http_session.use_ssl = uri.scheme.eql?('https')
-        # set http options in callback, such as timeout and cert. verification
+        # Set http options in callback, such as timeout and cert. verification
         RestParameters.instance.session_cb&.call(http_session)
-        # manually start session for keep alive (if supported by server, else, session is closed every time)
+        # Manually start session for keep alive (if supported by server, else, session is closed every time)
         http_session.start
         return http_session
       end
@@ -174,19 +186,31 @@ module Aspera
         return result
       end
 
-      # Parse a header string as returned by HTTP
-      # @param header [String] header string, e.g. "application/json; charset=utf-8"
-      # @return [Hash] parsed header with type and parameters
-      #   {type: 'application/json', parameters: {charset: 'utf-8'}}
+      # Parses an HTTP Content-Type header string into its media type and parameters
+      # according to RFC 9110 and RFC 6838.
+      # TODO: use gem: content_type
+      #
+      # @param header [String] The Content-Type header string, e.g., "application/json; charset=utf-8"
+      # @return [Hash] A hash with :type and :parameters keys.
+      #   Example:
+      #     {
+      #       type: "application/json",
+      #       parameters: {
+      #         charset: "utf-8",
+      #         version: "1.0"
+      #       }
+      #     }
       def parse_header(header)
-        type, *params = header.split(/;\s*/)
-        parameters = params.map do |param|
-          one = param.split(/=\s*/)
-          one[0] = one[0].to_sym
-          one[1] = one[1].gsub(/\A"|"\z/, '')
-          one
+        parts = header.split(';').map(&:strip)
+        media_type = parts.shift.downcase
+        parameters = parts.filter_map do |param|
+          key, value = param.split('=', 2)
+          next unless key && value
+          key = key.strip.downcase.to_sym
+          value = value.strip.gsub(/\A"|"\z/, '')
+          [key, value]
         end.to_h
-        {type: type.downcase, parameters: parameters}
+        {type: media_type, parameters: parameters}
       end
     end
 
@@ -200,8 +224,13 @@ module Aspera
 
     public
 
-    attr_reader :base_url
+    # All original constructor parameters
     attr_reader :auth_params
+
+    # The root URL for the API
+    attr_reader :base_url
+
+    # Base common headers of API
     attr_reader :headers
 
     # @return creation parameters
@@ -278,7 +307,7 @@ module Aspera
     # @param body [Hash, String] body parameters
     # @param headers [Hash] additional headers (override Content-Type)
     # @param save_to_file (filepath)
-    # @param return_error (bool)
+    # @param exception (bool) true, error raise exception
     def call(
       operation:,
       subpath: nil,
@@ -287,12 +316,14 @@ module Aspera
       body: nil,
       headers: nil,
       save_to_file: nil,
-      return_error: false
+      exception: true
     )
       subpath = subpath.to_s if subpath.is_a?(Symbol)
       subpath = '' if subpath.nil?
       Log.log.debug{"call #{operation} [#{subpath}]".red.bold.bg_green}
-      Log.dump(:body, body)
+      Log.dump(:body, body, level: :trace1)
+      Log.dump(:query, query, level: :trace1)
+      Log.dump(:headers, headers, level: :trace1)
       Aspera.assert_type(subpath, String)
       if headers.nil?
         headers = @headers.clone
@@ -462,12 +493,12 @@ module Aspera
             body: body,
             content_type: content_type,
             save_to_file: save_to_file,
-            return_error: return_error,
+            exception: exception,
             headers: headers
           )
         end
         # raise exception if could not retry and not return error in result
-        raise e unless return_error
+        raise e if exception
       end
       Log.log.debug{"result=http:#{result[:http]}, data:#{result[:data].class}"}
       return result
@@ -478,31 +509,36 @@ module Aspera
     # If specific elements are needed, then use the full `call` method
     #
 
-    def create(subpath, params)
-      return call(operation: 'POST', subpath: subpath, headers: {'Accept' => MIME_JSON}, body: params, content_type: MIME_JSON)[:data]
+    # Create: `POST`
+    def create(subpath, params, **kwargs)
+      return call(operation: 'POST', subpath: subpath, headers: {'Accept' => MIME_JSON}, body: params, content_type: MIME_JSON, **kwargs)[:data]
     end
 
-    def read(subpath, query = nil)
-      return call(operation: 'GET', subpath: subpath, headers: {'Accept' => MIME_JSON}, query: query)[:data]
+    # Read: `GET`
+    def read(subpath, query = nil, **kwargs)
+      return call(operation: 'GET', subpath: subpath, headers: {'Accept' => MIME_JSON}, query: query, **kwargs)[:data]
     end
 
-    def update(subpath, params)
-      return call(operation: 'PUT', subpath: subpath, headers: {'Accept' => MIME_JSON}, body: params, content_type: MIME_JSON)[:data]
+    # Update: `PUT`
+    def update(subpath, params, **kwargs)
+      return call(operation: 'PUT', subpath: subpath, headers: {'Accept' => MIME_JSON}, body: params, content_type: MIME_JSON, **kwargs)[:data]
     end
 
-    def delete(subpath, params = nil)
-      return call(operation: 'DELETE', subpath: subpath, headers: {'Accept' => MIME_JSON}, query: params)[:data]
+    # Delete: `DELETE`
+    def delete(subpath, params = nil, **kwargs)
+      return call(operation: 'DELETE', subpath: subpath, headers: {'Accept' => MIME_JSON}, query: params, **kwargs)[:data]
     end
 
-    def cancel(subpath)
-      return call(operation: 'CANCEL', subpath: subpath, headers: {'Accept' => MIME_JSON})[:data]
+    # Cancel: `CANCEL`
+    def cancel(subpath, **kwargs)
+      return call(operation: 'CANCEL', subpath: subpath, headers: {'Accept' => MIME_JSON}, **kwargs)[:data]
     end
 
     # Query entity by general search (read with parameter `q`)
     # TODO: not generic enough ? move somewhere ? inheritance ?
-    # @param subpath path of entity in API
-    # @param search_name name of searched entity
-    # @param query additional search query parameters
+    # @param subpath     [String] Path of entity in API
+    # @param search_name [String] Name of searched entity
+    # @param query       [Hash]   Additional search query parameters
     # @returns [Hash] A single entity matching the search, or an exception if not found or multiple found
     def lookup_by_name(subpath, search_name, query: nil)
       query = {} if query.nil?
@@ -520,10 +556,17 @@ module Aspera
         name_matches = matching_items.select{ |i| i['name'].casecmp?(search_name)}
         case name_matches.length
         when 1 then return name_matches.first
-        when 0 then raise %Q(#{subpath}: multiple case insensitive partial match for: "#{search_name}": #{matching_items.map{ |i| i['name']}} but no case insensitive full match. Please be more specific or give exact name.)
+        when 0 then raise %Q(#{subpath}: Multiple case insensitive partial match for: "#{search_name}": #{matching_items.map{ |i| i['name']}} but no case insensitive full match. Please be more specific or give exact name.)
         else raise "Two entities cannot have the same case insensitive name: #{name_matches.map{ |i| i['name']}}"
         end
       end
     end
+
+    # Content-Type that are JSON
+    JSON_DECODE = [MIME_JSON, 'application/vnd.api+json', 'application/x-javascript'].freeze
+
+    UNAVAILABLE_CODES = ['503']
+
+    private_constant :JSON_DECODE, :UNAVAILABLE_CODES
   end
 end

data/lib/aspera/ssh.rb

@@ -17,7 +17,7 @@ module Aspera
         $VERBOSE = nil
         Net::SSH::Authentication::Session.class_eval do
           define_method(:default_keys) do
-            %w[~/.ssh/id_dsa ~/.ssh/id_rsa ~/.ssh2/id_dsa ~/.ssh2/id_rsa].freeze
+            %w[.ssh .ssh2].product(%w[rsa dsa ecdsa]).map{"~/#{_1}/id_#{_2}"}.freeze
           end
           private(:default_keys)
         end rescue nil
@@ -81,6 +81,6 @@ module Aspera
   end
 end
 
-# HACK: deactivate ed25519 and ecdsa private keys from SSH identities, as it usually causes problems
-Aspera::Ssh.disable_ed25519_keys if ENV.fetch('ASCLI_ENABLE_ED25519', 'false').eql?('false')
+# Deactivate ed25519 and ecdsa private keys from SSH identities, as it usually causes problems
+Aspera::Ssh.disable_ed25519_keys if Gem::Specification.find_all_by_name('ed25519').none?
 Aspera::Ssh.disable_ecd_sha2_algorithms if defined?(JRUBY_VERSION) && ENV.fetch('ASCLI_ENABLE_ECDSHA2', 'false').eql?('false')