aspera-cli 4.24.2 → 4.25.0.pre

data/lib/aspera/oauth/base.rb

@@ -59,25 +59,13 @@ module Aspera
 
       attr_reader :scope, :api, :path_token, :client_id
 
-      # helper method to create token as per RFC
+      # Helper method to create token as per RFC
+      # @return [HTTPResponse]
+      # @raise RestError if not 2XX code
       def create_token_call(creation_params)
         Log.log.debug{'Generating a new token'.bg_green}
-        payload = if @use_query
-          {
-            query: creation_params
-          }
-        else
-          {
-            content_type: Rest::MIME_WWW,
-            body:         creation_params
-          }
-        end
-        return @api.call(
-          operation: 'POST',
-          subpath:   @path_token,
-          headers:   {'Accept' => Rest::MIME_JSON},
-          **payload
-        )
+        return @api.create(@path_token, nil, query: creation_params, ret: :resp) if @use_query
+        return @api.create(@path_token, creation_params, content_type: Rest::MIME_WWW, ret: :resp)
       end
 
       # @param add_secret [Boolean] Add secret in default call parameters
@@ -122,17 +110,18 @@ module Aspera
             Factory.instance.persist_mgr.delete(@token_cache_id)
             token_data = nil
             # lets try the existing refresh token
+            # NOTE: AoC admin token has no refresh, and lives by default 1800secs
             if !refresh_token.nil?
-              Log.log.info{"refresh=[#{refresh_token}]".bg_green}
-              # NOTE: AoC admin token has no refresh, and lives by default 1800secs
-              resp = create_token_call(optional_scope_client_id(add_secret: true).merge(grant_type: 'refresh_token', refresh_token: refresh_token))
-              if resp[:http].code.start_with?('2')
-                # save only if success
-                json_data = resp[:http].body
+              Log.log.info{"refresh token=[#{refresh_token}]".bg_green}
+              begin
+                http = create_token_call(optional_scope_client_id(add_secret: true).merge(grant_type: 'refresh_token', refresh_token: refresh_token))
+                # Save only if success
+                json_data = http.body
                 token_data = JSON.parse(json_data)
                 Factory.instance.persist_mgr.put(@token_cache_id, json_data)
-              else
-                Log.log.debug{"refresh failed: #{resp[:http].body}".bg_red}
+              rescue => e
+                # Refresh token can fail.
+                Log.log.warn{"Refresh failed: #{e}"}
               end
             end
           end
@@ -140,8 +129,9 @@ module Aspera
 
         # no cache, nor refresh: generate a token
         if token_data.nil?
-          resp = create_token
-          json_data = resp[:http].body
+          # Call the method-specific token creation
+          # which returns the result of create_token_call
+          json_data = create_token.body
           token_data = JSON.parse(json_data)
           Factory.instance.persist_mgr.put(@token_cache_id, json_data)
         end

data/lib/aspera/oauth/factory.rb

@@ -156,7 +156,7 @@ module Aspera
       def register_token_creator(creator_class)
         Aspera.assert_type(creator_class, Class)
         id = Factory.class_to_id(creator_class)
-        Log.log.debug{"registering token creator #{id}"}
+        Log.log.debug{"registering creator for #{id}"}
         @token_type_classes[id] = creator_class
       end
 

data/lib/aspera/oauth/url_json.rb

@@ -25,7 +25,8 @@ module Aspera
           query:        @query.merge(scope: scope), # scope is here because it may change over time (node)
           content_type: Rest::MIME_JSON,
           body:         @body,
-          headers:      {'Accept' => Rest::MIME_JSON}
+          headers:      {'Accept' => Rest::MIME_JSON},
+          ret:          :resp
         )
       end
     end

data/lib/aspera/preview/file_types.rb

@@ -3,7 +3,7 @@
 require 'aspera/log'
 require 'aspera/assert'
 require 'singleton'
-require 'mime/types'
+require 'marcel'
 
 module Aspera
   module Preview
@@ -61,7 +61,7 @@ module Aspera
       end
 
       # @param mimetype [String] mime type
-      # @return file type, one of enum CONVERSION_TYPES, or nil if not found
+      # @return [NilClass,Symbol] file type, one of enum CONVERSION_TYPES, or nil if not found
       def mime_to_type(mimetype)
         Aspera.assert_type(mimetype, String)
         return SUPPORTED_MIME_TYPES[mimetype] if SUPPORTED_MIME_TYPES.key?(mimetype)
@@ -72,19 +72,17 @@ module Aspera
         return
       end
 
-      # @param filepath [String] full path to file
-      # @param mimetype [String] provided by node API
+      # @param filepath [String] Full path to file
+      # @param mimetype [String] MIME typre provided by node API
       # @return file type, one of enum CONVERSION_TYPES
       # @raise [RuntimeError] if no conversion type found
       def conversion_type(filepath, mimetype)
         Log.log.debug{"conversion_type(#{filepath},mime=#{mimetype},magic=#{@use_mimemagic})"}
-        mimetype = nil if mimetype.is_a?(String) && (mimetype == 'application/octet-stream' || mimetype.empty?)
-        # Use mimemagic if available
-        mimetype ||= mime_using_mimemagic(filepath)
-        mimetype ||= mime_using_file(filepath)
-        # from extensions, using local mapping
-        mimetype ||= MIME::Types.of(File.basename(filepath)).first
-        raise "no MIME type found for #{File.basename(filepath)}" if mimetype.nil?
+        # Default type or empty means no type
+        mimetype = TYPE_NOT_FOUND if mimetype.nil? || (mimetype.is_a?(String) && mimetype.empty?)
+        mimetype = Marcel::MimeType.for(Pathname.new(filepath), name: File.basename(filepath), declared_type: mimetype)
+        mimetype = 'text/plain' if mimetype.eql?(TYPE_NOT_FOUND) && ascii_text_file?(filepath)
+        raise "no MIME type found for #{File.basename(filepath)}" if mimetype.eql?(TYPE_NOT_FOUND)
         conversion_type = mime_to_type(mimetype)
         raise "no conversion type found for #{File.basename(filepath)}" if conversion_type.nil?
         Log.log.trace1{"conversion_type(#{File.basename(filepath)}): #{conversion_type.class.name} [#{conversion_type}]"}
@@ -93,33 +91,21 @@ module Aspera
 
       private
 
-      # Use mime magic to find mime type based on file content (magic numbers)
-      # @param filepath [String] full path to file
-      # @return [String] mime type, or nil if not found
-      def mime_using_mimemagic(filepath)
-        return unless @use_mimemagic
-        # moved here, as `mimemagic` can cause installation issues
-        require 'mimemagic'
-        require 'mimemagic/version'
-        require 'mimemagic/overlay' if MimeMagic::VERSION.start_with?('0.3.')
-        # check magic number inside file (empty string if not found)
-        detected_mime = MimeMagic.by_magic(File.open(filepath)).to_s
-        # check extension only
-        if mime_to_type(detected_mime).nil?
-          Log.log.debug{"no conversion for #{detected_mime}, trying extension"}
-          detected_mime = MimeMagic.by_extension(File.extname(filepath)).to_s
-        end
-        detected_mime = nil if detected_mime.empty?
-        Log.log.debug{"mimemagic: #{detected_mime.class.name} [#{detected_mime}]"}
-        return detected_mime
-      end
+      TYPE_NOT_FOUND = 'application/octet-stream'
+      ACCEPT_CTRL_CHARS = [9, 10, 13]
 
-      # Use 'file' command to find mime type based on file content (Unix)
-      def mime_using_file(filepath)
-        return Environment.secure_capture(exec: 'file', args: ['--mime-type', '--brief', filepath]).strip
-      rescue => e
-        Log.log.error{"error using 'file' command: #{e.message}"}
-        return
+      # Returns true if the file looks like ASCII text (printable ASCII + \t, \r, \n, space).
+      # It reads only a small prefix (default: 64KB) and fails fast on the first bad byte.
+      def ascii_text_file?(path, sample_size: 64 * 1024)
+        File.open(path, 'rb') do |f|
+          sample = f.read(sample_size) || ''.b
+          sample.each_byte do |b|
+            next if b.between?(32, 126) || ACCEPT_CTRL_CHARS.include?(b)
+            # Any other control character => not ASCII text
+            return false
+          end
+          true
+        end
       end
     end
   end

data/lib/aspera/products/connect.rb

@@ -56,10 +56,10 @@ module Aspera
       # Retrieve structure from cloud (CDN) with all versions available
       def versions
         if @connect_versions.nil?
-          javascript = cdn_api.call(operation: 'GET', subpath: VERSION_INFO_FILE)
+          http = cdn_api.read(VERSION_INFO_FILE, ret: :resp)
           # get result on one line
-          connect_versions_javascript = javascript[:http].body.gsub(/\r?\n\s*/, '')
-          Log.log.debug{"javascript=[\n#{connect_versions_javascript}\n]"}
+          connect_versions_javascript = http.body.gsub(/\r?\n\s*/, '')
+          Log.dump(:javascript, connect_versions_javascript)
           # get javascript object only
           found = connect_versions_javascript.match(/^.*? = (.*);/)
           raise Cli::Error, 'Problem when getting connect versions from internet' if found.nil?

data/lib/aspera/rest.rb

@@ -50,13 +50,13 @@ module Aspera
     end
   end
 
+  class EntityNotFound < Error
+  end
+
   # a simple class to make HTTP calls, equivalent to rest-client
   # rest call errors are raised as exception RestCallError
   # and error are analyzed in RestErrorAnalyzer
   class Rest
-    # Error message when entity not found (TODO: use specific exception)
-    ENTITY_NOT_FOUND = 'No such'
-
     MIME_JSON = 'application/json'
     MIME_WWW = 'application/x-www-form-urlencoded'
     MIME_TEXT = 'text/plain'
@@ -71,7 +71,7 @@ module Aspera
       def basic_authorization(user, pass); return "Basic #{Base64.strict_encode64("#{user}:#{pass}")}"; end
 
       # Indicate that the given Hash query uses php style for array parameters
-      # a[]=1&a[]=2
+      # @param query [Hash] A key can have Array value and result will use PHP format: a[]=1&a[]=2
       def php_style(query)
         Aspera.assert_type(query, Hash){'query'}
         query[:x_array_php_style] = true
@@ -300,14 +300,16 @@ module Aspera
     end
 
     # HTTP/S REST call
-    # @param operation [String] HTTP operation (GET, POST, PUT, DELETE)
-    # @param subpath [String] subpath of REST API
-    # @param query [Hash] URL parameters
-    # @param content_type [String,nil] Type of body parameters (one of MIME_*) and serialization, else use headers
-    # @param body [Hash, String] body parameters
-    # @param headers [Hash] additional headers (override Content-Type)
-    # @param save_to_file (filepath)
-    # @param exception (bool) true, error raise exception
+    # @param operation    [String] HTTP operation (GET, POST, PUT, DELETE)
+    # @param subpath      [String] subpath of REST API
+    # @param query        [Hash] URL parameters
+    # @param content_type [String, nil] Type of body parameters (one of MIME_*) and serialization, else use headers
+    # @param body       [Hash, String] body parameters
+    # @param headers      [Hash] additional headers (override Content-Type)
+    # @param save_to_file [String, nil](filepath)
+    # @param exception    [Bool] `true`, error raise exception
+    # @param ret [:data, :resp, :both] Tell to return only data, only http response, or both
+    # @return [Object, Array] only data, only http response, or both
     def call(
       operation:,
       subpath: nil,
@@ -316,7 +318,8 @@ module Aspera
       body: nil,
       headers: nil,
       save_to_file: nil,
-      exception: true
+      exception: true,
+      ret: :data
     )
       subpath = subpath.to_s if subpath.is_a?(Symbol)
       subpath = '' if subpath.nil?
@@ -325,6 +328,8 @@ module Aspera
       Log.dump(:query, query, level: :trace1)
       Log.dump(:headers, headers, level: :trace1)
       Aspera.assert_type(subpath, String)
+      # We must have a way to check return code
+      Aspera.assert(exception || !ret.eql?(:data))
       if headers.nil?
         headers = @headers.clone
       else
@@ -348,7 +353,8 @@ module Aspera
         end
       else Aspera.error_unexpected_value(@auth_params[:type])
       end
-      result = {http: nil}
+      result_http = nil
+      result_data = nil
       # start a block to be able to retry the actual HTTP request in case of OAuth token expiration
       begin
         # TODO: shall we percent encode subpath (spaces) test with access key delete with space in id
@@ -391,14 +397,14 @@ module Aspera
         file_saved = false
         # make http request (pipelined)
         http_session.request(req) do |response|
-          result[:http] = response
-          result_mime = self.class.parse_header(result[:http]['Content-Type'] || MIME_TEXT)[:type]
-          Log.log.debug{"response: code=#{result[:http].code}, mime=#{result_mime}, mime2= #{response['Content-Type']}"}
+          result_http = response
+          result_mime = self.class.parse_header(result_http['Content-Type'] || MIME_TEXT)[:type]
+          Log.log.debug{"response: code=#{result_http.code}, mime=#{result_mime}, mime2= #{response['Content-Type']}"}
           # JSON data needs to be parsed, in case it contains an error code
           if !save_to_file.nil? &&
-              result[:http].code.to_s.start_with?('2') &&
+              result_http.code.to_s.start_with?('2') &&
               !JSON_DECODE.include?(result_mime)
-            total_size = result[:http]['Content-Length']&.to_i
+            total_size = result_http['Content-Length']&.to_i
             Log.log.debug('before write file')
             target_file = save_to_file
             # override user's path to path in header
@@ -416,7 +422,7 @@ module Aspera
             FileUtils.mkdir_p(File.dirname(target_file_tmp))
             limiter = TimerLimiter.new(0.5)
             File.open(target_file_tmp, 'wb') do |file|
-              result[:http].read_body do |fragment|
+              result_http.read_body do |fragment|
                 file.write(fragment)
                 written_size += fragment.length
                 RestParameters.instance.progress_bar&.event(:transfer, session_id: session_id, info: written_size) if limiter.trigger?
@@ -429,32 +435,32 @@ module Aspera
             file_saved = true
           end
         end
-        Log.log.debug{"result: code=#{result[:http].code} mime=#{result_mime}"}
+        Log.log.debug{"result: code=#{result_http.code} mime=#{result_mime}"}
         # sometimes there is a UTF8 char (e.g. (c) ), TODO : related to mime type encoding ?
-        # result[:http].body.force_encoding('UTF-8') if result[:http].body.is_a?(String)
-        # Log.log.debug{"result: body=#{result[:http].body}"}
+        # result_http.body.force_encoding('UTF-8') if result_http.body.is_a?(String)
+        # Log.log.debug{"result: body=#{result_http.body}"}
         case result_mime
         when *JSON_DECODE
-          result[:data] = JSON.parse(result[:http].body) rescue result[:http].body
-          Log.dump(:result_data, result[:data])
+          result_data = JSON.parse(result_http.body) rescue result_http.body
+          Log.dump(:result_data, result_data)
         else # when MIME_TEXT
-          result[:data] = result[:http].body
+          result_data = result_http.body
         end
-        RestErrorAnalyzer.instance.raise_on_error(req, result)
+        RestErrorAnalyzer.instance.raise_on_error(req, result_data, result_http)
         unless file_saved || save_to_file.nil?
           FileUtils.mkdir_p(File.dirname(save_to_file))
-          File.write(save_to_file, result[:http].body, binmode: true)
+          File.write(save_to_file, result_http.body, binmode: true)
         end
       rescue RestCallError => e
         do_retry = false
         # AoC have some timeout , like Connect to platform.bss.asperasoft.com:443 ...
         do_retry ||= true if e.response.body.include?('failed: connect timed out') && RestParameters.instance.retry_on_timeout
         # AoC sometimes not available
-        do_retry ||= true if RestParameters.instance.retry_on_unavailable && UNAVAILABLE_CODES.include?(result[:http].code.to_s)
+        do_retry ||= true if RestParameters.instance.retry_on_unavailable && UNAVAILABLE_CODES.include?(result_http.code.to_s)
         # possibility to retry anything if it fails
         do_retry ||= true if RestParameters.instance.retry_on_error
         # not authorized: oauth token expired
-        if @not_auth_codes.include?(result[:http].code.to_s) && @auth_params[:type].eql?(:oauth2)
+        if @not_auth_codes.include?(result_http.code.to_s) && @auth_params[:type].eql?(:oauth2)
           begin
             # try to use refresh token
             req['Authorization'] = oauth.authorization(refresh: true)
@@ -494,14 +500,20 @@ module Aspera
             content_type: content_type,
             save_to_file: save_to_file,
             exception: exception,
-            headers: headers
+            headers: headers,
+            ret: ret
           )
         end
         # raise exception if could not retry and not return error in result
         raise e if exception
       end
-      Log.log.debug{"result=http:#{result[:http]}, data:#{result[:data].class}"}
-      return result
+      Log.log.debug{"result=http:#{result_http}, data:#{result_data.class}"}
+      return case ret
+             when :data then result_data
+             when :resp then result_http
+             when :both then [result_data, result_http]
+             else Aspera.error_unexpected_value(ret){'Type of result for REST'}
+             end
     end
 
     #
@@ -511,27 +523,27 @@ module Aspera
 
     # Create: `POST`
     def create(subpath, params, **kwargs)
-      return call(operation: 'POST', subpath: subpath, headers: {'Accept' => MIME_JSON}, body: params, content_type: MIME_JSON, **kwargs)[:data]
+      return call(operation: 'POST', subpath: subpath, headers: {'Accept' => MIME_JSON}, body: params, content_type: MIME_JSON, **kwargs)
     end
 
     # Read: `GET`
     def read(subpath, query = nil, **kwargs)
-      return call(operation: 'GET', subpath: subpath, headers: {'Accept' => MIME_JSON}, query: query, **kwargs)[:data]
+      return call(operation: 'GET', subpath: subpath, headers: {'Accept' => MIME_JSON}, query: query, **kwargs)
     end
 
     # Update: `PUT`
     def update(subpath, params, **kwargs)
-      return call(operation: 'PUT', subpath: subpath, headers: {'Accept' => MIME_JSON}, body: params, content_type: MIME_JSON, **kwargs)[:data]
+      return call(operation: 'PUT', subpath: subpath, headers: {'Accept' => MIME_JSON}, body: params, content_type: MIME_JSON, **kwargs)
     end
 
     # Delete: `DELETE`
     def delete(subpath, params = nil, **kwargs)
-      return call(operation: 'DELETE', subpath: subpath, headers: {'Accept' => MIME_JSON}, query: params, **kwargs)[:data]
+      return call(operation: 'DELETE', subpath: subpath, headers: {'Accept' => MIME_JSON}, query: params, **kwargs)
     end
 
     # Cancel: `CANCEL`
     def cancel(subpath, **kwargs)
-      return call(operation: 'CANCEL', subpath: subpath, headers: {'Accept' => MIME_JSON}, **kwargs)[:data]
+      return call(operation: 'CANCEL', subpath: subpath, headers: {'Accept' => MIME_JSON}, **kwargs)
     end
 
     # Query entity by general search (read with parameter `q`)
@@ -549,7 +561,7 @@ module Aspera
       Aspera.assert_type(matching_items, Array)
       case matching_items.length
       when 1 then return matching_items.first
-      when 0 then raise %Q{#{ENTITY_NOT_FOUND} #{subpath}: "#{search_name}"}
+      when 0 then raise EntityNotFound, %Q{No such #{subpath}: "#{search_name}"}
       else
         # multiple case insensitive partial matches, try case insensitive full match
         # (anyway AoC does not allow creation of 2 entities with same case insensitive name)

data/lib/aspera/rest_error_analyzer.rb

@@ -27,13 +27,13 @@ module Aspera
     # Use this method to analyze a EST result and raise an exception
     # Analyzes REST call response and raises a RestCallError exception
     # if HTTP result code is not 2XX
-    def raise_on_error(req, res)
-      Log.log.debug{"raise_on_error #{req.method} #{req.path} #{res[:http].code}"}
+    def raise_on_error(req, data, http)
+      Log.log.debug{"raise_on_error #{req.method} #{req.path} #{http.code}"}
       call_context = {
         messages: [],
         request:  req,
-        response: res[:http],
-        data:     res[:data]
+        response: http,
+        data:     data
       }
       # multiple error messages can be found
       # analyze errors from provided handlers

data/lib/aspera/ssh.rb

@@ -11,6 +11,7 @@ module Aspera
     class Error < Aspera::Error
     end
     class << self
+      # HACK: disable some key type
       def disable_ed25519_keys
         Log.log.debug('Disabling SSH ed25519 user keys')
         old_verbose = $VERBOSE
@@ -24,6 +25,7 @@ module Aspera
         $VERBOSE = old_verbose
       end
 
+      # HACK: disable some algorithms
       def disable_ecd_sha2_algorithms
         Log.log.debug('Disabling SSH ecdsa')
         Net::SSH::Transport::Algorithms::ALGORITHMS.each_value{ |a| a.reject!{ |a| a =~ /^ecd(sa|h)-sha2/}}
@@ -38,6 +40,7 @@ module Aspera
       Aspera.assert_type(host, String)
       Aspera.assert_type(username, String)
       Aspera.assert_type(ssh_options, Hash)
+      ssh_options[:use_agent] = false unless ssh_options.key?(:use_agent)
       @host = host
       @username = username
       @ssh_options = ssh_options
@@ -62,7 +65,7 @@ module Aspera
             exit_code = data.read_long
             next if exit_code.zero?
             error_message = "#{cmd}: exit #{exit_code}, #{error.join.chomp}"
-            raise Error, error_message if  exception
+            raise Error, error_message if exception
             # Happens when windows user hasn't logged in and created home account.
             error_message += "\nHint: home not created in Windows?" if data.include?('Could not chdir to home directory')
             Log.log.debug(error_message)
@@ -82,5 +85,5 @@ module Aspera
 end
 
 # Deactivate ed25519 and ecdsa private keys from SSH identities, as it usually causes problems
-Aspera::Ssh.disable_ed25519_keys if Gem::Specification.find_all_by_name('ed25519').none?
+Aspera::Ssh.disable_ed25519_keys if Gem::Specification.find_all_by_name('ed25519').none? || ENV.fetch('ASCLI_ENABLE_ED25519', 'true').eql?('false')
 Aspera::Ssh.disable_ecd_sha2_algorithms if defined?(JRUBY_VERSION) && ENV.fetch('ASCLI_ENABLE_ECDSHA2', 'false').eql?('false')

data/lib/aspera/ssl.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+require 'openssl'
+require 'aspera/assert'
+require 'aspera/log'
+
+module Aspera
+  # Give possibility to globally override SSL options
+  module SSL
+    @extra_options = 0
+    class << self
+      @extra_options = OpenSSL::SSL::SSLContext::DEFAULT_PARAMS[:options]
+      attr_reader :extra_options
+
+      def option_list=(v)
+        Aspera.assert_type(v, Array){'ssl_options'}
+        v.each do |opt|
+          Aspera.assert_type(opt, String, Integer){'Expected String or Integer in ssl_options'}
+          case opt
+          when Integer
+            @extra_options = opt
+          when String
+            name = "OP_#{opt.start_with?('-') ? opt[1..] : opt}".upcase
+            raise Cli::BadArgument, "Unknown ssl_option: #{name}, use one of: #{OpenSSL::SSL.constants.grep(/^OP_/).map{ |c| c.to_s.sub(/^OP_/, '')}.join(', ')}" if !OpenSSL::SSL.const_defined?(name)
+            if opt.start_with?('-')
+              @extra_options &= ~OpenSSL::SSL.const_get(name)
+            else
+              @extra_options |= OpenSSL::SSL.const_get(name)
+            end
+          end
+        end
+      end
+    end
+    def set_params(params = {})
+      super(params)
+      self.options = Aspera::SSL.extra_options unless Aspera::SSL.extra_options.nil?
+      self
+    end
+  end
+end
+OpenSSL::SSL::SSLContext.prepend(Aspera::SSL)