aspera-cli 4.21.2 → 4.23.0

data/lib/aspera/preview/file_types.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require 'aspera/log'
+require 'aspera/assert'
 require 'singleton'
 require 'mime/types'
 
@@ -9,6 +10,7 @@ module Aspera
     # function conversion_type returns one of the types: CONVERSION_TYPES
     class FileTypes
       include Singleton
+
       # values for conversion_type : input format
       CONVERSION_TYPES = %i[image office pdf plaintext video].freeze
 
@@ -22,6 +24,8 @@ module Aspera
         'application/mxf'                    => :video,
         'application/mac-binhex40'           => :office,
         'application/msword'                 => :office,
+        'application/vnd.ms-excel'           => :office,
+        'application/vnd.ms-powerpoint'      => :office,
         'application/rtf'                    => :office,
         'application/x-abiword'              => :office,
         'application/x-mspublisher'          => :office,
@@ -56,17 +60,43 @@ module Aspera
       end
 
       # @param mimetype [String] mime type
-      # @return file type, one of enum CONVERSION_TYPES
+      # @return file type, one of enum CONVERSION_TYPES, or nil if not found
       def mime_to_type(mimetype)
+        Aspera.assert_type(mimetype, String)
         return SUPPORTED_MIME_TYPES[mimetype] if SUPPORTED_MIME_TYPES.key?(mimetype)
-        return :office if mimetype.start_with?('application/vnd.')
+        return :office if mimetype.start_with?('application/vnd.ms-')
+        return :office if mimetype.start_with?('application/vnd.openxmlformats-officedocument')
         return :video if mimetype.start_with?('video/')
         return :image if mimetype.start_with?('image/')
         return nil
       end
 
-      # use mime magic to find mime type based on file content (magic numbers)
-      def file_to_mime(filepath)
+      # @param filepath [String] full path to file
+      # @param mimetype [String] provided by node API
+      # @return file type, one of enum CONVERSION_TYPES
+      # @raise [RuntimeError] if no conversion type found
+      def conversion_type(filepath, mimetype)
+        Log.log.debug{"conversion_type(#{filepath},mime=#{mimetype},magic=#{@use_mimemagic})"}
+        mimetype = nil if mimetype.is_a?(String) && (mimetype == 'application/octet-stream' || mimetype.empty?)
+        # Use mimemagic if available
+        mimetype ||= mime_using_mimemagic(filepath)
+        mimetype ||= mime_using_file(filepath)
+        # from extensions, using local mapping
+        mimetype ||= MIME::Types.of(File.basename(filepath)).first
+        raise "no MIME type found for #{File.basename(filepath)}" if mimetype.nil?
+        conversion_type = mime_to_type(mimetype)
+        raise "no conversion type found for #{File.basename(filepath)}" if conversion_type.nil?
+        Log.log.trace1{"conversion_type(#{File.basename(filepath)}): #{conversion_type.class.name} [#{conversion_type}]"}
+        return conversion_type
+      end
+
+      private
+
+      # Use mime magic to find mime type based on file content (magic numbers)
+      # @param filepath [String] full path to file
+      # @return [String] mime type, or nil if not found
+      def mime_using_mimemagic(filepath)
+        return unless @use_mimemagic
         # moved here, as `mimemagic` can cause installation issues
         require 'mimemagic'
         require 'mimemagic/version'
@@ -83,35 +113,12 @@ module Aspera
         return detected_mime
       end
 
-      # @param filepath [String] full path to file
-      # @param mimetype [String] provided by node API
-      # @return file type, one of enum CONVERSION_TYPES
-      # @raise [RuntimeError] if no conversion type found
-      def conversion_type(filepath, mimetype)
-        Log.log.debug{"conversion_type(#{filepath},m=#{mimetype},t=#{@use_mimemagic})"}
-        # 1- get type from provided mime type, using local mapping
-        conversion_type = mime_to_type(mimetype) if !mimetype.nil?
-        # 2- else, from computed mime type (if available)
-        if conversion_type.nil? && @use_mimemagic
-          detected_mime = file_to_mime(filepath)
-          if !detected_mime.nil?
-            conversion_type = mime_to_type(detected_mime)
-            if !mimetype.nil?
-              if mimetype.eql?(detected_mime)
-                Log.log.debug('matching mime type per magic number')
-              else
-                # NOTE: detected can be nil
-                Log.log.debug{"non matching mime types: node=[#{mimetype}], magic=[#{detected_mime}]"}
-              end
-            end
-          end
-        end
-        # 3- else, from extensions, using local mapping
-        mime_by_ext = MIME::Types.of(File.basename(filepath)).first
-        conversion_type = mime_to_type(mime_by_ext.to_s) if conversion_type.nil? && !mime_by_ext.nil?
-        raise "no conversion type found for #{File.basename(filepath)}" if conversion_type.nil?
-        Log.log.trace1{"conversion_type(#{File.basename(filepath)}): #{conversion_type.class.name} [#{conversion_type}]"}
-        return conversion_type
+      # Use 'file' command to find mime type based on file content (Unix)
+      def mime_using_file(filepath)
+        return Environment.secure_capture(exec: 'file', args: ['--mime-type', '--brief', filepath]).strip
+      rescue => e
+        Log.log.error{"error using 'file' command: #{e.message}"}
+        return nil
       end
     end
   end

data/lib/aspera/preview/generator.rb

@@ -240,7 +240,7 @@ module Aspera
       # text to png
       def convert_plaintext_to_png
         # get 100 first lines of text file
-        first_lines = File.open(@source_file_path){|f|Array.new(100){f.readline rescue ''}.join}
+        first_lines = File.foreach(@source_file_path).first(100).join
         Utils.external_command(:magick, [
           'convert',
           '-size', "#{@options.thumb_img_size}x#{@options.thumb_img_size}",

data/lib/aspera/preview/options.rb

@@ -15,22 +15,22 @@ module Aspera
       # iw/ih : input width or height
       # -x : keep aspect ratio, having value a multiple of x
       DESCRIPTIONS = [
-        { name: :max_size,             default: 1 << 24,            description: 'maximum size (in bytes) of preview file' },
-        { name: :thumb_vid_scale,      default: "-1:'min(ih,100)'", description: 'png: video: size (ffmpeg scale argument)' },
-        { name: :thumb_vid_fraction,   default: 0.1,                description: 'png: video: time percent position of snapshot' },
-        { name: :thumb_img_size,       default: 800,                description: 'png: non-video: height (and width)' },
-        { name: :thumb_text_font,      default: 'Courier',          description: 'png: plaintext: font for text rendering: `magick identify -list font`'},
-        { name: :video_conversion,     default: :reencode,          description: 'mp4: method for preview generation', values: VIDEO_CONVERSION_METHODS },
-        { name: :video_png_conv,       default: :fixed,             description: 'mp4: method for thumbnail generation', values: VIDEO_THUMBNAIL_METHODS },
-        { name: :video_scale,          default: "'min(iw,360)':-2", description: 'mp4: all: video scale (ffmpeg scale argument)' },
-        { name: :video_start_sec,      default: 10,                 description: 'mp4: all: start offset (seconds) of video preview' },
-        { name: :reencode_ffmpeg,      default: {},                 description: 'mp4: reencode: options to ffmpeg' },
-        { name: :blend_keyframes,      default: 30,                 description: 'mp4: blend: # key frames' },
-        { name: :blend_pauseframes,    default: 3,                  description: 'mp4: blend: # pause frames' },
-        { name: :blend_transframes,    default: 5,                  description: 'mp4: blend: # transition blend frames' },
-        { name: :blend_fps,            default: 15,                 description: 'mp4: blend: frame per second' },
-        { name: :clips_count,          default: 5,                  description: 'mp4: clips: number of clips' },
-        { name: :clips_length,         default: 5,                  description: 'mp4: clips: length in seconds of each clips' }
+        {name: :max_size,             default: 1 << 24,            description: 'maximum size (in bytes) of preview file'},
+        {name: :thumb_vid_scale,      default: "-1:'min(ih,100)'", description: 'png: video: size (ffmpeg scale argument)'},
+        {name: :thumb_vid_fraction,   default: 0.1,                description: 'png: video: time percent position of snapshot'},
+        {name: :thumb_img_size,       default: 800,                description: 'png: non-video: height (and width)'},
+        {name: :thumb_text_font,      default: 'Courier',          description: 'png: plaintext: font for text rendering: `magick identify -list font`'},
+        {name: :video_conversion,     default: :reencode,          description: 'mp4: method for preview generation', values: VIDEO_CONVERSION_METHODS},
+        {name: :video_png_conv,       default: :fixed,             description: 'mp4: method for thumbnail generation', values: VIDEO_THUMBNAIL_METHODS},
+        {name: :video_scale,          default: "'min(iw,360)':-2", description: 'mp4: all: video scale (ffmpeg scale argument)'},
+        {name: :video_start_sec,      default: 10,                 description: 'mp4: all: start offset (seconds) of video preview'},
+        {name: :reencode_ffmpeg,      default: {},                 description: 'mp4: reencode: options to ffmpeg'},
+        {name: :blend_keyframes,      default: 30,                 description: 'mp4: blend: # key frames'},
+        {name: :blend_pauseframes,    default: 3,                  description: 'mp4: blend: # pause frames'},
+        {name: :blend_transframes,    default: 5,                  description: 'mp4: blend: # transition blend frames'},
+        {name: :blend_fps,            default: 15,                 description: 'mp4: blend: frame per second'},
+        {name: :clips_count,          default: 5,                  description: 'mp4: clips: number of clips'},
+        {name: :clips_length,         default: 5,                  description: 'mp4: clips: length in seconds of each clips'}
       ].freeze
       # add accessors
       DESCRIPTIONS.each do |opt|

data/lib/aspera/preview/terminal.rb

@@ -50,7 +50,7 @@ module Aspera
           pixel_colors = []
           image.each_pixel do |pixel, col, row|
             pixel_rgb = [pixel.red, pixel.green, pixel.blue]
-            pixel_rgb = pixel_rgb.map { |color| color >> shift_for_8_bit } unless shift_for_8_bit.eql?(0)
+            pixel_rgb = pixel_rgb.map{ |color| color >> shift_for_8_bit} unless shift_for_8_bit.eql?(0)
             # init 2-dim array
             pixel_colors[row] ||= []
             pixel_colors[row][col] = pixel_rgb
@@ -82,7 +82,7 @@ module Aspera
             size:                blob.length
             # width:               image.columns,
             # height:              image.rows
-          }.map { |k, v| "#{k}=#{v}" }.join(';')
+          }.map{ |k, v| "#{k}=#{v}"}.join(';')
           # \a is BEL, \e is ESC : https://github.com/ruby/ruby/blob/master/doc/syntax/literals.rdoc#label-Strings
           # escape sequence for iTerm2 image display
           return "\e]1337;File=#{arguments}:#{Base64.encode64(blob)}\a"
@@ -91,7 +91,7 @@ module Aspera
         # @return [Boolean] true if the terminal supports iTerm2 image display
         def iterm_supported?
           TERM_ENV_VARS.each do |env_var|
-            return true if ITERM_NAMES.any? { |term| ENV[env_var]&.include?(term) }
+            return true if ITERM_NAMES.any?{ |term| ENV[env_var]&.include?(term)}
           end
           false
         end

data/lib/aspera/preview/utils.rb

@@ -16,14 +16,18 @@ module Aspera
       # external binaries used
       EXTERNAL_TOOLS = %i[ffmpeg ffprobe magick optipng unoconv].freeze
       TEMP_FORMAT = 'img%04d.jpg'
+      FFMPEG_DEFAULT_PARAMS = [
+        '-y', # overwrite output without asking
+        '-loglevel', 'error' # show only errors and up
+      ].freeze
       private_constant :BASH_SPECIAL_CHARACTERS, :EXTERNAL_TOOLS, :TEMP_FORMAT
 
       class << self
         # returns string with single quotes suitable for bash if there is any bash meta-character
         def shell_quote(argument)
-          return argument unless argument.chars.any?{|c|BASH_SPECIAL_CHARACTERS.include?(c)}
+          return argument unless argument.chars.any?{ |c| BASH_SPECIAL_CHARACTERS.include?(c)}
           # surround with single quotes, and escape single quotes
-          return %Q{'#{argument.gsub("'"){|_s| %q{'"'"'}}}'}
+          return %Q{'#{argument.gsub("'"){ |_s| %q{'"'"'}}}'}
         end
 
         # check that external tools can be executed
@@ -53,17 +57,11 @@ module Aspera
           return Environment.secure_capture(exec: command_sym.to_s, args: command_args.map(&:to_s))
         end
 
-        def ffmpeg(a)
-          Aspera.assert_type(a, Hash)
-          # input_file,input_args,output_file,output_args
-          a[:gl_p] ||= [
-            '-y', # overwrite output without asking
-            '-loglevel', 'error' # show only errors and up
-          ]
-          a[:in_p] ||= []
-          a[:out_p] ||= []
-          Aspera.assert(%i[gl_p in_f in_p out_f out_p].eql?(a.keys.sort)){"wrong params (#{a.keys.sort})"}
-          external_command(:ffmpeg, [a[:gl_p], a[:in_p], '-i', a[:in_f], a[:out_p], a[:out_f]].flatten)
+        def ffmpeg(gl_p: FFMPEG_DEFAULT_PARAMS, in_p: [], in_f:, out_p: [], out_f:)
+          Aspera.assert_type(gl_p, Array)
+          Aspera.assert_type(in_p, Array)
+          Aspera.assert_type(out_p, Array)
+          external_command(:ffmpeg, gl_p +  in_p + ['-i', in_f] + out_p + [out_f])
         end
 
         # @return Float in seconds

data/lib/aspera/products/connect.rb

@@ -7,6 +7,7 @@ module Aspera
   module Products
     class Connect
       include Singleton
+
       APP_NAME = 'IBM Aspera Connect'
 
       class << self
@@ -43,7 +44,7 @@ module Aspera
             app_root: File.join(Dir.home, '.aspera', 'connect'),
             run_root: File.join(Dir.home, '.aspera', 'connect')
           }]
-          end.map { |i| i.merge({ expected: APP_NAME }) }
+          end.map{ |i| i.merge({expected: APP_NAME})}
         end
       end
 

data/lib/aspera/products/desktop.rb

@@ -18,7 +18,7 @@ module Aspera
             sub_bin:  File.join('Contents', 'Resources', 'transferd', 'bin')
           }]
           else []
-          end.map { |i| i.merge({ expected: APP_NAME }) }
+          end.map{ |i| i.merge({expected: APP_NAME})}
         end
 
         def log_file

data/lib/aspera/products/transferd.rb

@@ -15,7 +15,7 @@ module Aspera
           [{
             app_root: sdk_directory,
             sub_bin:  ''
-          }].map { |i| i.merge({ expected: APP_NAME }) }
+          }].map{ |i| i.merge({expected: APP_NAME})}
         end
 
         # location of SDK files

data/lib/aspera/proxy_auto_config.rb

@@ -13,7 +13,7 @@ module URI
       def register_proxy_finder
         Aspera.assert(block_given?)
         # overload the method in URI : call user's provided block and fallback to original method
-        define_method(:find_proxy) {|env_vars=ENV| yield(to_s) || find_proxy_orig(env_vars)}
+        define_method(:find_proxy){ |env_vars=ENV| yield(to_s) || find_proxy_orig(env_vars)}
       end
     end
   end
@@ -70,7 +70,7 @@ END_OF_JAVASCRIPT
     end
 
     def register_uri_generic
-      URI::Generic.register_proxy_finder{|url_str|get_proxies(url_str).first}
+      URI::Generic.register_proxy_finder{ |url_str| get_proxies(url_str).first}
       # allow chaining
       return self
     end

data/lib/aspera/rest.rb

@@ -6,12 +6,14 @@ require 'aspera/log'
 require 'aspera/assert'
 require 'aspera/oauth'
 require 'aspera/hash_ext'
+require 'aspera/timer_limiter'
 require 'net/http'
 require 'net/https'
 require 'json'
 require 'base64'
 require 'singleton'
 require 'securerandom'
+require 'fileutils'
 
 # Cancel method for HTTP
 class Net::HTTP::Cancel < Net::HTTPRequest # rubocop:disable Style/ClassAndModuleChildren
@@ -31,15 +33,18 @@ module Aspera
   class RestParameters
     include Singleton
 
-    attr_accessor :user_agent, :download_partial_suffix, :retry_on_error, :retry_sleep, :session_cb, :progress_bar
+    attr_accessor :user_agent, :download_partial_suffix, :retry_on_error, :retry_on_timeout, :retry_on_unavailable, :retry_max, :retry_sleep, :session_cb, :progress_bar
 
     private
 
     def initialize
       @user_agent = 'RubyAsperaRest'
       @download_partial_suffix = '.http_partial'
-      @retry_on_error = 0
-      @retry_sleep = nil
+      @retry_on_error = false
+      @retry_on_timeout = true
+      @retry_on_unavailable = true
+      @retry_max = 1
+      @retry_sleep = 4
       @session_cb = nil
       @progress_bar = nil
     end
@@ -57,8 +62,14 @@ module Aspera
     # error message when entity not found (TODO: use specific exception)
     ENTITY_NOT_FOUND = 'No such'
 
+    MIME_JSON = 'application/json'
+    MIME_WWW = 'application/x-www-form-urlencoded'
+    MIME_TEXT = 'text/plain'
+
     # Content-Type that are JSON
-    JSON_DECODE = ['application/json', 'application/vnd.api+json', 'application/x-javascript'].freeze
+    JSON_DECODE = [MIME_JSON, 'application/vnd.api+json', 'application/x-javascript'].freeze
+
+    UNAVAILABLE_CODES = ['503']
 
     class << self
       # @return [String] Basic auth token
@@ -98,7 +109,7 @@ module Aspera
             end
           end
         when Array
-          Aspera.assert(query.all?{|i| i.is_a?(Array) && i.length.eql?(2)}) {'Query must be array of arrays or 2 elements'}
+          Aspera.assert(query.all?{ |i| i.is_a?(Array) && i.length.eql?(2)}){'Query must be array of arrays or 2 elements'}
           query_array = query
         else
           raise "Query must be Hash or Array, not #{query.class}"
@@ -163,6 +174,10 @@ module Aspera
         return result
       end
 
+      # Parse a header string as returned by HTTP
+      # @param header [String] header string, e.g. "application/json; charset=utf-8"
+      # @return [Hash] parsed header with type and parameters
+      #   {type: 'application/json', parameters: {charset: 'utf-8'}}
       def parse_header(header)
         type, *params = header.split(/;\s*/)
         parameters = params.map do |param|
@@ -171,7 +186,7 @@ module Aspera
           one[1] = one[1].gsub(/\A"|"\z/, '')
           one
         end.to_h
-        { type: type.downcase, parameters: parameters }
+        {type: type.downcase, parameters: parameters}
       end
     end
 
@@ -189,6 +204,7 @@ module Aspera
 
     attr_reader :base_url
     attr_reader :auth_params
+    attr_reader :headers
 
     # @return creation parameters
     def params
@@ -238,7 +254,7 @@ module Aspera
       @http_session = nil
       @redirect_max = redirect_max
       Aspera.assert_type(@redirect_max, Integer)
-      @headers = headers
+      @headers = headers.clone
       Aspera.assert_type(@headers, Hash)
       @headers['User-Agent'] ||= RestParameters.instance.user_agent
       # OAuth object (created on demand)
@@ -249,7 +265,7 @@ module Aspera
     def oauth
       if @oauth.nil?
         Aspera.assert(@auth_params[:type].eql?(:oauth2)){'no OAuth defined'}
-        oauth_parameters = @auth_params.reject { |k, _v| k.eql?(:type) }
+        oauth_parameters = @auth_params.reject{ |k, _v| k.eql?(:type)}
         Log.log.debug{Log.dump('oauth parameters', oauth_parameters)}
         @oauth = OAuth::Factory.instance.create(**oauth_parameters)
       end
@@ -260,20 +276,20 @@ module Aspera
     # @param operation [String] HTTP operation (GET, POST, PUT, DELETE)
     # @param subpath [String] subpath of REST API
     # @param query [Hash] URL parameters
+    # @param content_type [String,nil] Type of body parameters (one of MIME_*) and serialization, else use headers
     # @param body [Hash, String] body parameters
-    # @param body_type [Symbol] type of body parameters (:json, :www, :text, nil)
+    # @param headers [Hash] additional headers (override Content-Type)
     # @param save_to_file (filepath)
     # @param return_error (bool)
-    # @param headers [Hash] additional headers
     def call(
       operation:,
       subpath: nil,
       query: nil,
+      content_type: nil,
       body: nil,
-      body_type: nil,
+      headers: nil,
       save_to_file: nil,
-      return_error: false,
-      headers: nil
+      return_error: false
     )
       subpath = subpath.to_s if subpath.is_a?(Symbol)
       subpath = '' if subpath.nil?
@@ -317,19 +333,18 @@ module Aspera
         rescue NameError
           raise "unsupported operation : #{operation}"
         end
-        case body_type
-        when :json
+        case content_type
+        when nil # ignore
+        when MIME_JSON
           req.body = JSON.generate(body) # , ascii_only: true
-          req['Content-Type'] = 'application/json'
-        when :www
+          req['Content-Type'] = MIME_JSON
+        when MIME_WWW
           req.body = URI.encode_www_form(body)
-          req['Content-Type'] = 'application/x-www-form-urlencoded'
-        when :text
+          req['Content-Type'] = MIME_WWW
+        when MIME_TEXT
           req.body = body
-          req['Content-Type'] = 'text/plain'
-        when nil
-        else
-          raise "unsupported body type : #{body_type}"
+          req['Content-Type'] = MIME_TEXT
+        else Aspera.error_unexpected_value(content_type){'body type'}
         end
         # set headers
         headers.each do |key, value|
@@ -338,10 +353,8 @@ module Aspera
         # :type = :basic
         req.basic_auth(@auth_params[:username], @auth_params[:password]) if @auth_params[:type].eql?(:basic)
         Log.log.trace1{Log.dump(:req_body, req.body)}
-        # we try the call, and will retry only if oauth, as we can, first with refresh, and then re-auth if refresh is bad
-        oauth_tries ||= 2
-        timeout_tries ||= 5
-        general_tries ||= 1 + RestParameters.instance.retry_on_error
+        # we try the call, and will retry on some error types
+        error_tries ||= 1 + RestParameters.instance.retry_max
         # initialize with number of initial retries allowed, nil gives zero
         tries_remain_redirect = @redirect_max if tries_remain_redirect.nil?
         Log.log.debug("send request (retries=#{tries_remain_redirect})")
@@ -350,19 +363,19 @@ module Aspera
         # make http request (pipelined)
         http_session.request(req) do |response|
           result[:http] = response
-          result_mime = self.class.parse_header(result[:http]['Content-Type'] || 'text/plain')[:type]
+          result_mime = self.class.parse_header(result[:http]['Content-Type'] || MIME_TEXT)[:type]
+          Log.log.debug{"response: code=#{result[:http].code}, mime=#{result_mime}, mime2= #{response['Content-Type']}"}
           # JSON data needs to be parsed, in case it contains an error code
           if !save_to_file.nil? &&
               result[:http].code.to_s.start_with?('2') &&
-              !result[:http]['Content-Length'].nil? &&
               !JSON_DECODE.include?(result_mime)
-            total_size = result[:http]['Content-Length'].to_i
+            total_size = result[:http]['Content-Length']&.to_i
             Log.log.debug('before write file')
             target_file = save_to_file
             # override user's path to path in header
             if !response['Content-Disposition'].nil?
               disposition = self.class.parse_header(response['Content-Disposition'])
-              if disposition[:parameters].key?(:filename)
+              if disposition[:parameters].key?(:filename) && !disposition[:parameters][:filename].eql?('.')
                 target_file = File.join(File.dirname(target_file), disposition[:parameters][:filename])
               end
             end
@@ -372,12 +385,14 @@ module Aspera
             written_size = 0
             session_id = SecureRandom.uuid.freeze
             RestParameters.instance.progress_bar&.event(:session_start, session_id: session_id)
-            RestParameters.instance.progress_bar&.event(:session_size, session_id: session_id, info: total_size)
+            RestParameters.instance.progress_bar&.event(:session_size, session_id: session_id, info: total_size) if total_size
+            FileUtils.mkdir_p(File.dirname(target_file_tmp))
+            limiter = TimerLimiter.new(0.5)
             File.open(target_file_tmp, 'wb') do |file|
               result[:http].read_body do |fragment|
                 file.write(fragment)
                 written_size += fragment.length
-                RestParameters.instance.progress_bar&.event(:transfer, session_id: session_id, info: written_size)
+                RestParameters.instance.progress_bar&.event(:transfer, session_id: session_id, info: written_size) if limiter.trigger?
               end
             end
             RestParameters.instance.progress_bar&.event(:end, session_id: session_id)
@@ -394,17 +409,22 @@ module Aspera
         when *JSON_DECODE
           result[:data] = JSON.parse(result[:http].body) rescue result[:http].body
           Log.log.debug{Log.dump('result_data', result[:data])}
-        else # when 'text/plain'
+        else # when MIME_TEXT
           result[:data] = result[:http].body
         end
         RestErrorAnalyzer.instance.raise_on_error(req, result)
-        File.write(save_to_file, result[:http].body, binmode: true) unless file_saved || save_to_file.nil?
+        unless file_saved || save_to_file.nil?
+          FileUtils.mkdir_p(File.dirname(save_to_file))
+          File.write(save_to_file, result[:http].body, binmode: true)
+        end
       rescue RestCallError => e
         do_retry = false
         # AoC have some timeout , like Connect to platform.bss.asperasoft.com:443 ...
-        do_retry = true if e.response.body.include?('failed: connect timed out') && (timeout_tries -= 1).positive?
+        do_retry ||= true if e.response.body.include?('failed: connect timed out') && RestParameters.instance.retry_on_timeout
+        # AoC sometimes not available
+        do_retry ||= true if RestParameters.instance.retry_on_unavailable && UNAVAILABLE_CODES.include?(result[:http].code.to_s)
         # possibility to retry anything if it fails
-        do_retry = true if (general_tries -= 1).positive?
+        do_retry ||= true if RestParameters.instance.retry_on_error
         # not authorized: oauth token expired
         if @not_auth_codes.include?(result[:http].code.to_s) && @auth_params[:type].eql?(:oauth2)
           begin
@@ -417,10 +437,10 @@ module Aspera
             req['Authorization'] = oauth.authorization(cache: false)
           end
           Log.log.debug{"using new token=#{headers['Authorization']}"}
-          do_retry = true if (oauth_tries -= 1).positive?
+          do_retry ||= true
         end
-        if do_retry
-          sleep(RestParameters.instance.retry_sleep) unless RestParameters.instance.retry_sleep.nil?
+        if do_retry && (error_tries -= 1).positive?
+          sleep(RestParameters.instance.retry_sleep) unless RestParameters.instance.retry_sleep.eql?(0)
           retry
         end
         # redirect ? (any code beginning with 3)
@@ -436,13 +456,13 @@ module Aspera
           end
           # forwards the request to the new location
           return self.class.new(base_url: new_url, redirect_max: tries_remain_redirect).call(
-            operation: operation, query: query, body: body, body_type: body_type,
+            operation: operation, query: query, body: body, content_type: content_type,
             save_to_file: save_to_file, return_error: return_error, headers: headers)
         end
         # raise exception if could not retry and not return error in result
         raise e unless return_error
       end
-      Log.log.debug{"result=#{result}"}
+      Log.log.debug{"result=http:#{result[:http]}, data:#{result[:data].class}"}
       return result
     end
 
@@ -452,23 +472,23 @@ module Aspera
     #
 
     def create(subpath, params)
-      return call(operation: 'POST', subpath: subpath, headers: {'Accept' => 'application/json'}, body: params, body_type: :json)[:data]
+      return call(operation: 'POST', subpath: subpath, headers: {'Accept' => MIME_JSON}, body: params, content_type: MIME_JSON)[:data]
     end
 
     def read(subpath, query=nil)
-      return call(operation: 'GET', subpath: subpath, headers: {'Accept' => 'application/json'}, query: query)[:data]
+      return call(operation: 'GET', subpath: subpath, headers: {'Accept' => MIME_JSON}, query: query)[:data]
     end
 
     def update(subpath, params)
-      return call(operation: 'PUT', subpath: subpath, headers: {'Accept' => 'application/json'}, body: params, body_type: :json)[:data]
+      return call(operation: 'PUT', subpath: subpath, headers: {'Accept' => MIME_JSON}, body: params, content_type: MIME_JSON)[:data]
     end
 
     def delete(subpath, params=nil)
-      return call(operation: 'DELETE', subpath: subpath, headers: {'Accept' => 'application/json'}, query: params)[:data]
+      return call(operation: 'DELETE', subpath: subpath, headers: {'Accept' => MIME_JSON}, query: params)[:data]
     end
 
     def cancel(subpath)
-      return call(operation: 'CANCEL', subpath: subpath, headers: {'Accept' => 'application/json'})[:data]
+      return call(operation: 'CANCEL', subpath: subpath, headers: {'Accept' => MIME_JSON})[:data]
     end
 
     # Query entity by general search (read with parameter `q`)
@@ -490,11 +510,11 @@ module Aspera
       else
         # multiple case insensitive partial matches, try case insensitive full match
         # (anyway AoC does not allow creation of 2 entities with same case insensitive name)
-        name_matches = matching_items.select{|i|i['name'].casecmp?(search_name)}
+        name_matches = matching_items.select{ |i| i['name'].casecmp?(search_name)}
         case name_matches.length
         when 1 then return name_matches.first
-        when 0 then raise %Q(#{subpath}: multiple case insensitive partial match for: "#{search_name}": #{matching_items.map{|i|i['name']}} but no case insensitive full match. Please be more specific or give exact name.) # rubocop:disable Layout/LineLength
-        else raise "Two entities cannot have the same case insensitive name: #{name_matches.map{|i|i['name']}}"
+        when 0 then raise %Q(#{subpath}: multiple case insensitive partial match for: "#{search_name}": #{matching_items.map{ |i| i['name']}} but no case insensitive full match. Please be more specific or give exact name.) # rubocop:disable Layout/LineLength
+        else raise "Two entities cannot have the same case insensitive name: #{name_matches.map{ |i| i['name']}}"
         end
       end
     end

data/lib/aspera/rest_error_analyzer.rb

@@ -8,6 +8,7 @@ module Aspera
   # analyze error codes returned by REST calls and raise ruby exception
   class RestErrorAnalyzer
     include Singleton
+
     attr_accessor :log_file
 
     # the singleton object is registered with application specific handlers

data/lib/aspera/rest_errors_aspera.rb

@@ -35,7 +35,7 @@ module Aspera
           d_t_s.each do |res|
             r_err = res.dig(*%w[transfer_spec error]) || res['error']
             next unless r_err.is_a?(Hash)
-            RestErrorAnalyzer.add_error(call_context, type, "#{r_err['code']}: #{r_err['reason']}: #{r_err['user_message']}")
+            RestErrorAnalyzer.add_error(call_context, type, r_err.values.join(': '))
           end
         end
         RestErrorAnalyzer.instance.add_simple_handler(name: 'T9:IBM cloud IAM', path: ['errorMessage'])