aspera-cli 4.19.0 → 4.20.0

data/lib/aspera/rest.rb

@@ -11,6 +11,8 @@ require 'net/https'
 require 'json'
 require 'base64'
 require 'cgi'
+require 'singleton'
+require 'securerandom'
 
 # Cancel method for HTTP
 class Net::HTTP::Cancel < Net::HTTPRequest # rubocop:disable Style/ClassAndModuleChildren
@@ -20,22 +22,32 @@ class Net::HTTP::Cancel < Net::HTTPRequest # rubocop:disable Style/ClassAndModul
 end
 
 module Aspera
+  # Global settings
+  # @param user_agent [String] HTTP request header: 'User-Agent'
+  # @param download_partial_suffix [String] suffix for partial download
+  # @param session_cb [lambda] lambda called on new HTTP session. Takes the Net::HTTP as arg. Used to change parameters on creation.
+  # @param progress_bar [Object] progress bar object
+  class RestParameters
+    include Singleton
+
+    attr_accessor :user_agent, :download_partial_suffix, :retry_on_error, :retry_sleep, :session_cb, :progress_bar
+
+    private
+
+    def initialize
+      @user_agent = 'RubyAsperaRest'
+      @download_partial_suffix = '.http_partial'
+      @retry_on_error = 0
+      @retry_sleep = nil
+      @session_cb = nil
+      @progress_bar = nil
+    end
+  end
+
   # a simple class to make HTTP calls, equivalent to rest-client
   # rest call errors are raised as exception RestCallError
   # and error are analyzed in RestErrorAnalyzer
   class Rest
-    # Global settings also valid for any subclass
-    # @param user_agent [String] HTTP request header: 'User-Agent'
-    # @param download_partial_suffix [String] suffix for partial download
-    # @param session_cb [lambda] lambda called on new HTTP session. Takes the Net::HTTP as arg. Used to change parameters on creation.
-    # @param progress_bar [Object] progress bar object
-    @@global = { # rubocop:disable Style/ClassVars
-      user_agent:              'RubyAsperaRest',
-      download_partial_suffix: '.http_partial',
-      session_cb:              nil,
-      progress_bar:            nil
-    }
-
     # flag for array parameters prefixed with []
     ARRAY_PARAMS = '[]'
 
@@ -61,37 +73,45 @@ module Aspera
         return values.first.eql?(ARRAY_PARAMS)
       end
 
-      # Build URI from URL and parameters and check it is http or https, encode array [] parameters
-      def build_uri(url, query_hash=nil)
+      # Build URI from URL and parameters and check it is http or https
+      # encode array [] parameters
+      # @param query [Hash,Array]
+      def build_uri(url, query=nil)
         uri = URI.parse(url)
         Aspera.assert(%w[http https].include?(uri.scheme)){"REST endpoint shall be http/s not #{uri.scheme}"}
-        return uri if query_hash.nil?
-        Log.log.debug{Log.dump('query', query_hash)}
-        Aspera.assert_type(query_hash, Hash)
-        return uri if query_hash.empty?
-        query = []
-        query_hash.each do |k, v|
-          case v
-          when Array
-            # support array for query parameter, there is no standard. Either p[]=1&p[]=2, or p=1&p=2
-            suffix = array_params?(v) ? v.shift : ''
-            v.each do |e|
-              query.push(["#{k}#{suffix}", e])
+        return uri if query.nil? || query.respond_to?(:empty?) && query.empty?
+        Log.log.debug{Log.dump('query', query)}
+        query_array = []
+        case query
+        when Hash
+          query.each do |k, v|
+            case v
+            when Array
+              # support array for query parameter, there is no standard. Either p[]=1&p[]=2, or p=1&p=2
+              suffix = array_params?(v) ? v.shift : ''
+              v.each do |e|
+                query_array.push(["#{k}#{suffix}", e])
+              end
+            else
+              query_array.push([k, v])
             end
-          else
-            query.push([k, v])
           end
+        when Array
+          Aspera.assert(query.all?{|i| i.is_a?(Array) && i.length.eql?(2)}) {'Query must be array of arrays or 2 elements'}
+          query_array = query
+        else
+          raise "Query must be Hash or Array, not #{query.class}"
         end
         # [] is allowed in url parameters
-        uri.query = URI.encode_www_form(query).gsub('%5B%5D=', '[]=')
+        uri.query = URI.encode_www_form(query_array).gsub('%5B%5D=', '[]=')
         return uri
       end
 
-      # decode query string as hash
+      # Decode query string as Hash
       # Does not support arrays in query string, no standard, e.g. PHP's way is p[]=1&p[]=2
-      # @param query [String] query string
+      # @param query [String] query string as in URI.query
       # @return [Hash] decoded query
-      def decode_query(query)
+      def query_to_h(query)
         URI.decode_www_form(query).each_with_object({}) do |pair, h|
           key = pair.first
           raise "Array not supported in query string: #{key}" if key.include?('[]') || h.key?(key)
@@ -108,7 +128,7 @@ module Aspera
         http_session = Net::HTTP.new(uri.host, uri.port)
         http_session.use_ssl = uri.scheme.eql?('https')
         # set http options in callback, such as timeout and cert. verification
-        @@global[:session_cb]&.call(http_session)
+        RestParameters.instance.session_cb&.call(http_session)
         # manually start session for keep alive (if supported by server, else, session is closed every time)
         http_session.start
         return http_session
@@ -142,19 +162,6 @@ module Aspera
         return result
       end
 
-      # set global parameters
-      def set_parameters(**options)
-        options.each do |key, value|
-          Aspera.assert(@@global.key?(key)){"Unknown Rest option #{key}"}
-          @@global[key] = value
-        end
-      end
-
-      # @return [String] HTTP agent name
-      def user_agent
-        return @@global[:user_agent]
-      end
-
       def parse_header(header)
         type, *params = header.split(/;\s*/)
         parameters = params.map do |param|
@@ -179,16 +186,17 @@ module Aspera
 
     public
 
-    attr_reader :auth_params
     attr_reader :base_url
+    attr_reader :auth_params
 
+    # @return creation parameters
     def params
       return {
-        base_url:       @base_url,
-        auth:           @auth_params,
-        not_auth_codes: @not_auth_codes,
-        redirect_max:   @redirect_max,
-        headers:        @headers
+        base_url:       @base_url,       # String
+        auth:           @auth_params,    # Hash
+        not_auth_codes: @not_auth_codes, # Array
+        redirect_max:   @redirect_max,   # Integer
+        headers:        @headers         # Hash
       }
     end
 
@@ -226,9 +234,10 @@ module Aspera
       # OAuth object (created on demand)
       @oauth = nil
       @redirect_max = redirect_max
+      Aspera.assert_type(@redirect_max, Integer)
       @headers = headers.nil? ? {} : headers
       Aspera.assert_type(@headers, Hash)
-      @headers['User-Agent'] ||= @@global[:user_agent]
+      @headers['User-Agent'] ||= RestParameters.instance.user_agent
     end
 
     # @return the OAuth object (create, or cached if already created)
@@ -263,6 +272,8 @@ module Aspera
     )
       subpath = subpath.to_s if subpath.is_a?(Symbol)
       subpath = '' if subpath.nil?
+      Log.log.debug{"#{operation} [#{subpath}]".red.bold.bg_green}
+      Log.log.debug{Log.dump(:body, body)}
       Aspera.assert_type(subpath, String)
       if headers.nil?
         headers = @headers.clone
@@ -272,7 +283,6 @@ module Aspera
         headers.merge!(h)
       end
       Aspera.assert_type(headers, Hash)
-      Log.log.debug{"#{operation} [#{subpath}]".red.bold.bg_green}
       case @auth_params[:type]
       when :none
         # no auth
@@ -322,11 +332,13 @@ module Aspera
         end
         # :type = :basic
         req.basic_auth(@auth_params[:username], @auth_params[:password]) if @auth_params[:type].eql?(:basic)
-        Log.log.debug{Log.dump(:req_body, req.body)}
+        Log.log.trace1{Log.dump(:req_body, req.body)}
         # we try the call, and will retry only if oauth, as we can, first with refresh, and then re-auth if refresh is bad
         oauth_tries ||= 2
+        timeout_tries ||= 5
+        general_tries ||= 1 + RestParameters.instance.retry_on_error
         # initialize with number of initial retries allowed, nil gives zero
-        tries_remain_redirect = @redirect_max.to_i if tries_remain_redirect.nil?
+        tries_remain_redirect = @redirect_max if tries_remain_redirect.nil?
         Log.log.debug("send request (retries=#{tries_remain_redirect})")
         result_mime = nil
         file_saved = false
@@ -350,38 +362,44 @@ module Aspera
               end
             end
             # download with temp filename
-            target_file_tmp = "#{target_file}#{@@global[:download_partial_suffix]}"
+            target_file_tmp = "#{target_file}#{RestParameters.instance.download_partial_suffix}"
             Log.log.debug{"saving to: #{target_file}"}
             written_size = 0
-            @@global[:progress_bar]&.event(session_id: 1, type: :session_start)
-            @@global[:progress_bar]&.event(session_id: 1, type: :session_size, info: total_size)
+            session_id = SecureRandom.uuid.freeze
+            RestParameters.instance.progress_bar&.event(:session_start, session_id: session_id)
+            RestParameters.instance.progress_bar&.event(:session_size, session_id: session_id, info: total_size)
             File.open(target_file_tmp, 'wb') do |file|
               result[:http].read_body do |fragment|
                 file.write(fragment)
                 written_size += fragment.length
-                @@global[:progress_bar]&.event(session_id: 1, type: :transfer, info: written_size)
+                RestParameters.instance.progress_bar&.event(:transfer, session_id: session_id, info: written_size)
               end
             end
-            @@global[:progress_bar]&.event(session_id: 1, type: :end)
+            RestParameters.instance.progress_bar&.event(:end, session_id: session_id)
             # rename at the end
             File.rename(target_file_tmp, target_file)
             file_saved = true
           end
         end
+        Log.log.debug{"result: code=#{result[:http].code} mime=#{result_mime}"}
         # sometimes there is a UTF8 char (e.g. (c) ), TODO : related to mime type encoding ?
         # result[:http].body.force_encoding('UTF-8') if result[:http].body.is_a?(String)
         # Log.log.debug{"result: body=#{result[:http].body}"}
-        result[:data] = case result_mime
+        case result_mime
         when *JSON_DECODE
-          JSON.parse(result[:http].body) rescue result[:http].body
+          result[:data] = JSON.parse(result[:http].body) rescue result[:http].body
+          Log.log.debug{Log.dump('result_data', result[:data])}
         else # when 'text/plain'
-          result[:http].body
+          result[:data] = result[:http].body
         end
-        Log.log.debug{"result: code=#{result[:http].code} mime=#{result_mime}"}
-        Log.log.debug{Log.dump('data', result[:data])}
         RestErrorAnalyzer.instance.raise_on_error(req, result)
-        File.write(save_to_file, result[:http].body) unless file_saved || save_to_file.nil?
+        File.write(save_to_file, result[:http].body, binmode: true) unless file_saved || save_to_file.nil?
       rescue RestCallError => e
+        do_retry = false
+        # AoC have some timeout , like Connect to platform.bss.asperasoft.com:443 ...
+        do_retry = true if e.response.body.include?('failed: connect timed out') && (timeout_tries -= 1).positive?
+        # possibility to retry anything if it fails
+        do_retry = true if (general_tries -= 1).positive?
         # not authorized: oauth token expired
         if @not_auth_codes.include?(result[:http].code.to_s) && @auth_params[:type].eql?(:oauth2)
           begin
@@ -394,7 +412,11 @@ module Aspera
             req['Authorization'] = oauth.token(refresh: true)
           end
           Log.log.debug{"using new token=#{headers['Authorization']}"}
-          retry if (oauth_tries -= 1).nonzero?
+          do_retry = true if (oauth_tries -= 1).positive?
+        end
+        if do_retry
+          sleep(RestParameters.instance.retry_sleep) unless RestParameters.instance.retry_sleep.nil?
+          retry
         end
         # redirect ? (any code beginning with 3)
         if e.response.is_a?(Net::HTTPRedirection) && tries_remain_redirect.positive?
@@ -420,27 +442,28 @@ module Aspera
     end
 
     #
-    # CRUD methods here
+    # CRUD simplified methods here
+    # If specific elements are needed, then use the full `call` method
     #
 
     def create(subpath, params)
-      return call(operation: 'POST', subpath: subpath, headers: {'Accept' => 'application/json'}, body: params, body_type: :json)
+      return call(operation: 'POST', subpath: subpath, headers: {'Accept' => 'application/json'}, body: params, body_type: :json)[:data]
     end
 
     def read(subpath, query=nil)
-      return call(operation: 'GET', subpath: subpath, headers: {'Accept' => 'application/json'}, query: query)
+      return call(operation: 'GET', subpath: subpath, headers: {'Accept' => 'application/json'}, query: query)[:data]
     end
 
     def update(subpath, params)
-      return call(operation: 'PUT', subpath: subpath, headers: {'Accept' => 'application/json'}, body: params, body_type: :json)
+      return call(operation: 'PUT', subpath: subpath, headers: {'Accept' => 'application/json'}, body: params, body_type: :json)[:data]
     end
 
     def delete(subpath, params=nil)
-      return call(operation: 'DELETE', subpath: subpath, headers: {'Accept' => 'application/json'}, query: params)
+      return call(operation: 'DELETE', subpath: subpath, headers: {'Accept' => 'application/json'}, query: params)[:data]
     end
 
     def cancel(subpath)
-      return call(operation: 'CANCEL', subpath: subpath, headers: {'Accept' => 'application/json'})
+      return call(operation: 'CANCEL', subpath: subpath, headers: {'Accept' => 'application/json'})[:data]
     end
 
     # Query entity by general search (read with parameter `q`)
@@ -449,9 +472,10 @@ module Aspera
     # @param search_name name of searched entity
     # @param query additional search query parameters
     # @returns [Hash] A single entity matching the search, or an exception if not found or multiple found
-    def lookup_by_name(subpath, search_name, query={})
+    def lookup_by_name(subpath, search_name, query: nil)
+      query = {} if query.nil?
       # returns entities matching the query (it matches against several fields in case insensitive way)
-      matching_items = read(subpath, query.merge({'q' => search_name}))[:data]
+      matching_items = read(subpath, query.merge({'q' => search_name}))
       # API style: {totalcount:, ...} cspell: disable-line
       matching_items = matching_items[subpath] if matching_items.is_a?(Hash)
       Aspera.assert_type(matching_items, Array)

data/lib/aspera/transfer/faux_file.rb

@@ -13,16 +13,16 @@ module Aspera
         # @return nil if not a faux: scheme, else a FauxFile instance
         def create(name)
           return nil unless name.start_with?(PREFIX)
-          url_parts = name[PREFIX.length..-1].split('?')
-          raise 'Format: #{PREFIX}<file path>?<size>' unless url_parts.length.eql?(2)
-          raise "Format: <integer>[#{SUFFIX.join(',')}]" unless (m = url_parts[1].downcase.match(/^(\d+)([#{SUFFIX.join('')}])$/))
+          name_params = name[PREFIX.length..-1].split('?', 2)
+          raise 'Format: #{PREFIX}<file path>?<size>' unless name_params.length.eql?(2)
+          raise "Format: <integer>[#{SUFFIX.join(',')}]" unless (m = name_params[1].downcase.match(/^(\d+)([#{SUFFIX.join('')}])$/))
           size = m[1].to_i
           suffix = m[2]
           SUFFIX.each do |s|
             size *= 1024
             break if s.eql?(suffix)
           end
-          return FauxFile.new(url_parts[0], size)
+          return FauxFile.new(name_params[0], size)
         end
       end
       attr_reader :path, :size

data/lib/aspera/transfer/parameters.rb

@@ -29,10 +29,21 @@ module Aspera
       private_constant :SUPPORTED_AGENTS, :FILE_LIST_OPTIONS
 
       class << self
+        # temp file list files are created here
+        def file_list_folder=(value)
+          @file_list_folder = value
+          return if @file_list_folder.nil?
+
+          FileUtils.mkdir_p(@file_list_folder)
+          TempFileManager.instance.cleanup_expired(@file_list_folder)
+        end
+
         # Temp folder for file lists, must contain only file lists
         # because of garbage collection takes any file there
-        # this could be refined, as , for instance, on macos, temp folder is already user specific
-        @file_list_folder = TempFileManager.instance.new_file_path_global('asession_filelists') # cspell:disable-line
+        # this could be refined, as, for example, on macos, temp folder is already user specific
+        def file_list_folder
+          @file_list_folder ||= TempFileManager.instance.new_file_path_global('asession_filelists')
+        end
 
         # @param formatter [Cli::Formatter] formatter to use
         # @return a table suitable to display in manual
@@ -67,9 +78,7 @@ module Aspera
               else
                 param[:d].eql?(tick_yes) ? '' : 'n/a'
               end
-            if options.key?(:enum)
-              param[:description] += "\nAllowed values: #{options[:enum].join(', ')}"
-            end
+            param[:description] += "\nAllowed values: #{options[:enum].join(', ')}" if options.key?(:enum)
             # replace "solidus" HTML entity with its text value
             param[:description] = param[:description].gsub('&sol;', '\\')
             result.push(param)
@@ -91,17 +100,6 @@ module Aspera
         def ascp_args_file_list?(ascp_args)
           ascp_args&.any?{|i|FILE_LIST_OPTIONS.include?(i)}
         end
-
-        # temp file list files are created here
-        def file_list_folder=(value)
-          @file_list_folder = value
-          return if @file_list_folder.nil?
-          FileUtils.mkdir_p(@file_list_folder)
-          TempFileManager.instance.cleanup_expired(@file_list_folder)
-        end
-
-        # static methods
-        attr_reader :file_list_folder
       end
 
       # @param options [Hash] key: :wss: bool, :ascp_args: array of strings

data/lib/aspera/transfer/spec.rb

@@ -10,34 +10,30 @@ module Aspera
     class Spec
       # default transfer username for access key based transfers
       ACCESS_KEY_TRANSFER_USER = 'xfer'
+      # default ports for SSH and UDP
       SSH_PORT = 33_001
       UDP_PORT = 33_001
+      # base transfer spec for access keys
       AK_TSPEC_BASE = {
         'remote_user' => ACCESS_KEY_TRANSFER_USER,
         'ssh_port'    => SSH_PORT,
         'fasp_port'   => UDP_PORT
       }.freeze
-      # fields for transport
-      TRANSPORT_FIELDS = %w[remote_host remote_user ssh_port fasp_port wss_enabled wss_port].freeze
+      # fields for WSS
+      WSS_FIELDS = %w[wss_enabled wss_port].freeze
+      # all fields for transport
+      TRANSPORT_FIELDS = %w[remote_host remote_user ssh_port fasp_port].concat(WSS_FIELDS).freeze
       # reserved tag for Aspera
       TAG_RESERVED = 'aspera'
       class << self
         # translate upload/download to send/receive
         def transfer_type_to_direction(transfer_type)
-          case transfer_type.to_sym
-          when :upload then DIRECTION_SEND
-          when :download then DIRECTION_RECEIVE
-          else Aspera.error_unexpected_value(transfer_type.to_sym)
-          end
+          XFER_TYPE_TO_DIR.fetch(transfer_type)
         end
 
         # translate send/receive to upload/download
         def direction_to_transfer_type(direction)
-          case direction
-          when DIRECTION_SEND then :upload
-          when DIRECTION_RECEIVE then :download
-          else Aspera.error_unexpected_value(direction)
-          end
+          XFER_DIR_TO_TYPE.fetch(direction)
         end
       end
       DESCRIPTION = CommandLineBuilder.normalize_description(YAML.load_file("#{__FILE__[0..-3]}yaml"))
@@ -49,6 +45,10 @@ module Aspera
           const_set("#{name.to_s.upcase}_#{enum.upcase.gsub(/[^A-Z0-9]/, '_')}", enum.freeze)
         end
       end
+      # DIRECTION_* are read from yaml
+      XFER_TYPE_TO_DIR = {upload: DIRECTION_SEND, download: DIRECTION_RECEIVE}.freeze
+      XFER_DIR_TO_TYPE = XFER_TYPE_TO_DIR.invert.freeze
+      private_constant :XFER_TYPE_TO_DIR, :XFER_DIR_TO_TYPE
     end
   end
 end

data/lib/aspera/transfer/sync.rb

@@ -266,11 +266,7 @@ module Aspera
           else
             raise 'At least one of `local` or `sessions` must be present in async parameters'
           end
-          Environment.secure_spawn(env: {}, exec: ASYNC_ADMIN_EXECUTABLE, args: arguments, log_only: true)
-          stdout, stderr, status = Open3.capture3(*[ASYNC_ADMIN_EXECUTABLE].concat(arguments))
-          Log.log.debug{"status=#{status}, stderr=#{stderr}"}
-          Log.log.trace1{"stdout=#{stdout}"}
-          raise "Sync failed: #{status.exitstatus} : #{stderr}" unless status.success?
+          stdout = Environment.secure_capture(exec: ASYNC_ADMIN_EXECUTABLE, args: arguments)
           return parse_status(stdout)
         end
       end

data/lib/aspera/transfer/uri.rb

@@ -26,7 +26,7 @@ module Aspera
         # faspex 4 does not encode trailing base64 padding, fix that to be able to decode properly
         fixed_query = @fasp_uri.query.gsub(/(=+)$/){|trail_equals|'%3D' * trail_equals.length}
 
-        Rest.decode_query(fixed_query).each do |name, value|
+        Rest.query_to_h(fixed_query).each do |name, value|
           case name
           when 'cookie'      then result_ts['cookie'] = value
           when 'token'       then result_ts['token'] = value

data/lib/aspera/uri_reader.rb

@@ -12,7 +12,7 @@ module Aspera
         uri = URI.parse(uri_to_read)
         case uri.scheme
         when 'http', 'https'
-          return Rest.new(base_url: uri_to_read, redirect_max: 5).call(operation: 'GET', subpath: '', headers: {'Accept' => 'text/plain'})[:data]
+          return Rest.new(base_url: uri_to_read, redirect_max: 5).call(operation: 'GET', headers: {'Accept' => '*/*'})[:data]
         when 'file', NilClass
           local_file_path = uri.path
           raise 'URL shall have a path, check syntax' if local_file_path.nil?