aspera-cli 4.27.0 → 4.27.2

data/lib/aspera/rest.rb

@@ -87,7 +87,7 @@ module Aspera
       # @param query [Hash] A key can have Array value and result will use PHP format: a[]=1&a[]=2
       # @return [Hash] The query parameters.
       def php_style(query)
-        Aspera.assert_type(query, Hash){'query'}
+        Aspera.assert_type(query, Hash) { 'query' }
         query[:x_array_php_style] = true
         query
       end
@@ -100,7 +100,7 @@ module Aspera
       # @return [URI] The built URI.
       def build_uri(url, query)
         uri = URI.parse(url)
-        Aspera.assert_values(uri.scheme, %w[http https]){'URI scheme'}
+        Aspera.assert_values(uri.scheme, %w[http https]) { 'URI scheme' }
         return uri if query.nil? || query.respond_to?(:empty?) && query.empty?
         Log.dump(:query, query)
         uri.query =
@@ -110,9 +110,9 @@ module Aspera
           when Hash
             URI.encode_www_form(h_to_query_array(query))
           when Array
-            Aspera.assert(query.all?{ |i| i.is_a?(Array) && i.length.eql?(2)}, 'Query must be array of arrays of 2 elements')
+            Aspera.assert(query.all? { |i| i.is_a?(Array) && i.length.eql?(2) }, 'Query must be array of arrays of 2 elements')
             URI.encode_www_form(query) # remove nil values
-          else Aspera.error_unexpected_value(query.class){'query type'}
+          else Aspera.error_unexpected_value(query.class) { 'query type' }
           end.gsub('%5B%5D=', '[]=')
         # [] is allowed in url parameters
         uri
@@ -165,7 +165,7 @@ module Aspera
       # @return [Net::HTTP] A started HTTP session
       def start_http_session(base_url)
         uri = URI.parse(base_url)
-        Aspera.assert_values(uri.scheme, %w[http https]){'URI scheme'}
+        Aspera.assert_values(uri.scheme, %w[http https]) { 'URI scheme' }
         # This honors http_proxy env var
         http_session = Net::HTTP.new(uri.host, uri.port)
         http_session.use_ssl = uri.scheme.eql?('https')
@@ -185,7 +185,7 @@ module Aspera
         Aspera.assert_type(http_session, Net::HTTP)
         # Net::BufferedIO in net/protocol.rb
         result = http_session.instance_variable_get(:@socket)
-        Aspera.assert(!result.nil?){"no socket for #{http_session}"}
+        Aspera.assert(!result.nil?) { "no socket for #{http_session}" }
         return result
       end
 
@@ -290,7 +290,7 @@ module Aspera
       # remove trailing port if it is 443 and scheme is https
       @base_url = @base_url.gsub(/:443$/, '') if @base_url.start_with?('https://')
       @base_url = @base_url.gsub(/:80$/, '') if @base_url.start_with?('http://')
-      Log.log.debug{"Rest.new(#{@base_url})"}
+      Log.log.debug { "Rest.new(#{@base_url})" }
       # default is no auth
       @auth_params = auth
       Aspera.assert_type(@auth_params, Hash)
@@ -312,7 +312,7 @@ module Aspera
     def oauth
       if @oauth.nil?
         Aspera.assert(@auth_params[:type].eql?(:oauth2), 'no OAuth defined')
-        oauth_parameters = @auth_params.reject{ |k, _v| k.eql?(:type)}
+        oauth_parameters = @auth_params.reject { |k, _v| k.eql?(:type) }
         Log.dump(:oauth_parameters, oauth_parameters)
         @oauth = OAuth::Factory.instance.create(**oauth_parameters)
       end
@@ -326,7 +326,7 @@ module Aspera
     # @param content_type [String, nil] Type of body parameters (one of MIME_*) and serialization, else use headers
     # @param body         [Hash, String, nil] Body parameters
     # @param headers      [Hash{String => String}] Additional headers (override Content-Type)
-    # @param save_to_file [String, nil] File path to save response body
+    # @param save_to      [String, IO, nil] File path or IO object to save response body; progress bar is used when set
     # @param exception    [Boolean] Whether to raise an exception on HTTP error
     # @param ret          [Symbol] One of :data, :resp, :both - controls return value
     # @return [Array(Hash, Net::HTTPResponse)] When `ret` is :both
@@ -340,42 +340,20 @@ module Aspera
       content_type: nil,
       body: nil,
       headers: nil,
-      save_to_file: nil,
+      save_to: nil,
       exception: true,
       ret: :data
     )
       subpath = subpath.to_s if subpath.is_a?(Symbol)
       subpath = '' if subpath.nil?
-      Log.log.debug{"call #{operation} [#{subpath}]".red.bold.bg_green}
+      Log.log.debug { "call #{operation} [#{subpath}]".red.bold.bg_green }
       Log.dump(:body, body, level: :trace1)
       Log.dump(:query, query, level: :trace1)
       Log.dump(:headers, headers, level: :trace1)
       Aspera.assert_type(subpath, String)
       # We must have a way to check return code
       Aspera.assert(exception || !ret.eql?(:data), 'ret: :data requires exception handler')
-      if headers.nil?
-        headers = @headers.clone
-      else
-        h = headers
-        headers = @headers.clone
-        headers.merge!(h)
-      end
-      Aspera.assert_type(headers, Hash)
-      case @auth_params[:type]
-      when :none
-        # no auth
-      when :basic
-        Log.log.debug('using Basic auth')
-        # done in build_req
-      when :oauth2
-        headers['Authorization'] = oauth.authorization unless headers.key?('Authorization')
-      when :url
-        query ||= {}
-        @auth_params[:url_query].each do |key, value|
-          query[key] = value
-        end
-      else Aspera.error_unexpected_value(@auth_params[:type])
-      end
+      headers, query = prepare_call(headers, query)
       result_http = nil
       result_data = nil
       # initialize with number of initial retries allowed, nil gives zero
@@ -383,37 +361,7 @@ module Aspera
       # start a block to be able to retry the actual HTTP request in case of OAuth token expiration
       begin
         Log.log.debug("send request (retries=#{tries_remain_redirect})")
-        # TODO: shall we percent encode subpath (spaces) test with access key delete with space in id
-        # URI.escape()
-        separator = ['', '/'].include?(subpath) ? '' : '/'
-        uri = self.class.build_uri("#{@base_url}#{separator}#{subpath}", query)
-        Log.log.debug{"URI=#{uri}"}
-        begin
-          # instantiate request object based on string name
-          req = Net::HTTP.const_get(operation.capitalize).new(uri)
-        rescue NameError
-          raise "unsupported operation : #{operation}"
-        end
-        case content_type
-        when nil # ignore
-        when Mime::JSON
-          req.body = JSON.generate(body) # , ascii_only: true
-          req['Content-Type'] = Mime::JSON
-        when Mime::WWW
-          req.body = URI.encode_www_form(body)
-          req['Content-Type'] = Mime::WWW
-        when Mime::TEXT
-          req.body = body
-          req['Content-Type'] = Mime::TEXT
-        else Aspera.error_unexpected_value(content_type){'body type'}
-        end
-        # set headers
-        headers.each do |key, value|
-          req[key] = value
-        end
-        # :type = :basic
-        req.basic_auth(@auth_params[:username], @auth_params[:password]) if @auth_params[:type].eql?(:basic)
-        Log.dump(:req_body, req.body, level: :trace1)
+        req = build_request(operation, subpath, query, content_type, body, headers)
         # we try the call, and will retry on some error types
         error_tries ||= 1 + RestParameters.instance.retry_max
         result_mime = nil
@@ -422,58 +370,21 @@ module Aspera
         http_session.request(req) do |response|
           result_http = response
           result_mime = self.class.parse_header(result_http['Content-Type'] || Mime::TEXT)[:type]
-          Log.log.debug{"response: code=#{result_http.code}, mime=#{result_mime}, content-type=#{response['Content-Type']}"}
+          Log.log.debug { "response: code=#{result_http.code}, mime=#{result_mime}, content-type=#{response['Content-Type']}" }
           # JSON data needs to be parsed, in case it contains an error code
-          if !save_to_file.nil? &&
-              result_http.code.to_s.start_with?('2') &&
-              !Mime.json?(result_mime)
-            total_size = result_http['Content-Length']&.to_i
-            Log.log.debug('before write file')
-            target_file = save_to_file
-            # override user's path to path in header
-            if !response['Content-Disposition'].nil?
-              disposition = self.class.parse_header(response['Content-Disposition'])
-              target_file = File.join(File.dirname(target_file), disposition[:parameters][:filename]) if disposition[:parameters].key?(:filename) && !disposition[:parameters][:filename].eql?('.')
-            end
-            # download with temp filename
-            target_file_tmp = "#{target_file}#{RestParameters.instance.download_partial_suffix}"
-            Log.log.debug{"saving to: #{target_file}"}
-            written_size = 0
-            session_id = SecureRandom.uuid.freeze
-            RestParameters.instance.progress_bar&.event(:session_start, session_id: session_id)
-            RestParameters.instance.progress_bar&.event(:session_size, session_id: session_id, info: total_size) if total_size
-            FileUtils.mkdir_p(File.dirname(target_file_tmp))
-            limiter = TimerLimiter.new(0.5)
-            File.open(target_file_tmp, 'wb') do |file|
-              result_http.read_body do |fragment|
-                file.write(fragment)
-                written_size += fragment.length
-                RestParameters.instance.progress_bar&.event(:transfer, session_id: session_id, info: written_size) if limiter.trigger?
-              end
-            end
-            RestParameters.instance.progress_bar&.event(:session_end, session_id: session_id)
-            RestParameters.instance.progress_bar&.event(:end)
-            # rename at the end
-            File.rename(target_file_tmp, target_file)
-            file_saved = true
-          end
+          file_saved = save_response(response, result_http, result_mime, save_to)
         end
-        Log.log.debug{"result: code=#{result_http.code} mime=#{result_mime}"}
+        Log.log.debug { "result: code=#{result_http.code} mime=#{result_mime}" }
         # sometimes there is a UTF8 char (e.g. (c) )
         # TODO : related to mime type encoding ?
         # result_http.body.force_encoding('UTF-8') if result_http.body.is_a?(String)
         # Log.log.debug{"result: body=#{result_http.body}"}
-        result_data = result_http.body
-        Log.dump(:result_data_raw, result_data, level: :trace1)
-        # TODO: Remove next 2 lines when bug in async node api is fixed. (Aspera/core/issues/4490)
-        node_api_bug = result_data&.index('}HTTP/1.1 400 Bad Request') if result_data.is_a?(String)
-        result_data = result_data[0..node_api_bug] if node_api_bug
-        result_data = JSON.parse(result_data) if Mime.json?(result_mime) && !result_data.nil? && !result_data.empty?
-        Log.dump(:result_data, result_data)
+        result_data = parse_response(result_http, result_mime)
         RestErrorAnalyzer.instance.raise_on_error(req, result_data, result_http)
-        unless file_saved || save_to_file.nil?
-          FileUtils.mkdir_p(File.dirname(save_to_file))
-          File.write(save_to_file, result_http.body, binmode: true)
+        unless file_saved || save_to.nil?
+          raise 'save_to: IO object requires a streaming response' if save_to.respond_to?(:write)
+          FileUtils.mkdir_p(File.dirname(save_to))
+          File.write(save_to, result_http.body, binmode: true)
         end
       rescue RestCallError => e
         do_retry = false
@@ -494,7 +405,7 @@ module Aspera
             # regenerate a brand new token
             req['Authorization'] = oauth.authorization(cache: false)
           end
-          Log.log.debug{"using new token=#{headers['Authorization']}"}
+          Log.log.debug('using new token')
           do_retry ||= true
         end
         if do_retry && (error_tries -= 1).positive?
@@ -509,7 +420,7 @@ module Aspera
           # special case: relative redirect
           if URI.parse(new_url).host.nil?
             # we don't manage relative redirects with non-absolute path
-            Aspera.assert(new_url.start_with?('/')){"redirect location is relative: #{new_url}, but does not start with /."}
+            Aspera.assert(new_url.start_with?('/')) { "redirect location is relative: #{new_url}, but does not start with /." }
             new_url = "#{current_uri.scheme}://#{current_uri.host}#{new_url}"
           end
           # forwards the request to the new location
@@ -522,7 +433,7 @@ module Aspera
             query: query,
             body: body,
             content_type: content_type,
-            save_to_file: save_to_file,
+            save_to: save_to,
             exception: exception,
             headers: headers,
             ret: ret
@@ -531,15 +442,138 @@ module Aspera
         # raise exception if could not retry and not return error in result
         raise e if exception
       end
-      Log.log.debug{"result=http:#{result_http}, data:#{result_data.class}"}
+      Log.log.debug { "result=http:#{result_http}, data:#{result_data.class}" }
       return case ret
              when :data then result_data
              when :resp then result_http
              when :both then [result_data, result_http]
-             else Aspera.error_unexpected_value(ret){'Type of result for REST'}
+             else Aspera.error_unexpected_value(ret) { 'Type of result for REST' }
              end
     end
 
+    private
+
+    def prepare_call(headers, query)
+      if headers.nil?
+        headers = @headers.clone
+      else
+        h = headers
+        headers = @headers.clone
+        headers.merge!(h)
+      end
+      Aspera.assert_type(headers, Hash)
+      case @auth_params[:type]
+      when :none
+        # no auth
+      when :basic
+        Log.log.debug('using Basic auth')
+        # done in build_req
+      when :oauth2
+        headers['Authorization'] = oauth.authorization unless headers.key?('Authorization')
+      when :url
+        query ||= {}
+        @auth_params[:url_query].each do |key, value|
+          query[key] = value
+        end
+      else Aspera.error_unexpected_value(@auth_params[:type])
+      end
+      [headers, query]
+    end
+
+    def build_request(operation, subpath, query, content_type, body, headers)
+      # TODO: shall we percent encode subpath (spaces) test with access key delete with space in id
+      # URI.escape()
+      separator = ['', '/'].include?(subpath) ? '' : '/'
+      uri = self.class.build_uri("#{@base_url}#{separator}#{subpath}", query)
+      Log.log.debug { "URI=#{uri}" }
+      begin
+        # instantiate request object based on string name
+        req = Net::HTTP.const_get(operation.capitalize).new(uri)
+      rescue NameError
+        raise "unsupported operation : #{operation}"
+      end
+      case content_type
+      when nil # ignore
+      when Mime::JSON
+        req.body = JSON.generate(body) # , ascii_only: true
+        req['Content-Type'] = Mime::JSON
+      when Mime::WWW
+        req.body = URI.encode_www_form(body)
+        req['Content-Type'] = Mime::WWW
+      when Mime::TEXT
+        req.body = body
+        req['Content-Type'] = Mime::TEXT
+      else Aspera.error_unexpected_value(content_type) { 'body type' }
+      end
+      # set headers
+      headers.each do |key, value|
+        req[key] = value
+      end
+      # :type = :basic
+      req.basic_auth(@auth_params[:username], @auth_params[:password]) if @auth_params[:type].eql?(:basic)
+      Log.dump(:req_body, req.body, level: :trace1)
+      req
+    end
+
+    def save_response(response, result_http, result_mime, save_to)
+      return false unless !save_to.nil? && result_http.code.to_s.start_with?('2') && !Mime.json?(result_mime)
+
+      total_size = result_http['Content-Length']&.to_i
+      Log.log.debug('before write file')
+      target_file = save_to
+      # override user's path to path in header
+      unless response['Content-Disposition'].nil?
+        disposition = self.class.parse_header(response['Content-Disposition'])
+        if disposition[:parameters].key?(:filename) && !disposition[:parameters][:filename].eql?('.')
+          # Use only the basename to prevent path traversal via a server-controlled Content-Disposition header
+          safe_filename = File.basename(disposition[:parameters][:filename])
+          target_file = File.join(File.dirname(target_file), safe_filename) unless safe_filename.empty?
+        end
+      end
+      Log.log.debug { "saving to: #{target_file}" }
+      written_size = 0
+      session_id = SecureRandom.uuid.freeze
+      RestParameters.instance.progress_bar&.event(:session_start, session_id: session_id)
+      RestParameters.instance.progress_bar&.event(:session_size, session_id: session_id, info: total_size) if total_size
+      limiter = TimerLimiter.new(0.5)
+      if target_file.respond_to?(:write)
+        # IO object: stream directly into it
+        result_http.read_body do |fragment|
+          target_file.write(fragment)
+          written_size += fragment.length
+          RestParameters.instance.progress_bar&.event(:transfer, session_id: session_id, info: written_size) if limiter.trigger?
+        end
+      else
+        # file path: download to partial name first, then rename atomically
+        target_file_tmp = "#{target_file}#{RestParameters.instance.download_partial_suffix}"
+        FileUtils.mkdir_p(File.dirname(target_file_tmp))
+        File.open(target_file_tmp, 'wb') do |file|
+          result_http.read_body do |fragment|
+            file.write(fragment)
+            written_size += fragment.length
+            RestParameters.instance.progress_bar&.event(:transfer, session_id: session_id, info: written_size) if limiter.trigger?
+          end
+        end
+        File.rename(target_file_tmp, target_file)
+      end
+      RestParameters.instance.progress_bar&.event(:session_end, session_id: session_id)
+      RestParameters.instance.progress_bar&.event(:end)
+      true
+    end
+
+    def parse_response(result_http, result_mime)
+      result_data = result_http.body
+      Log.dump(:result_data_raw, result_data, level: :trace1)
+      # TODO: Remove next 2 lines when bug in async node api is fixed. (Aspera/core/issues/4490)
+      node_api_bug = result_data&.index('}HTTP/1.1 400 Bad Request') if result_data.is_a?(String)
+      result_data = result_data[0..node_api_bug] if node_api_bug
+      result_data = JSON.parse(result_data) if Mime.json?(result_mime) && !result_data.nil? && !result_data.empty?
+      Log.dump(:result_data, result_data)
+      result_data
+    end
+
+    public
+
     #
     # CRUD simplified methods here
     # If specific elements are needed, then use the full `call` method

data/lib/aspera/rest_error_analyzer.rb

@@ -36,7 +36,7 @@ module Aspera
     # @param data [Object]            the parsed response body
     # @param http [Net::HTTPResponse] the raw HTTP response
     def raise_on_error(req, data, http)
-      Log.log.debug{"raise_on_error #{req.method} #{req.path} #{http.code}"}
+      Log.log.debug { "raise_on_error #{req.method} #{req.path} #{http.code}" }
       context = {
         messages: [],
         request:  req,
@@ -49,7 +49,7 @@ module Aspera
       @error_handlers.each do |handler|
         handler[:block].call(handler[:name], context)
       rescue StandardError => e
-        Log.log.error{"ERROR in handler:\n#{e.message}\n#{e.backtrace}"}
+        Log.log.error { "ERROR in handler:\n#{e.message}\n#{e.backtrace}" }
       end
       raise RestCallError, context unless context[:messages].empty?
     end
@@ -97,7 +97,7 @@ module Aspera
       # @param message [String] one error message  to add to list
       def add_error(context, type, message)
         context[:messages].push(message)
-        Log.log.trace1{"Found error: #{type}: #{message}"}
+        Log.log.trace1 { "Found error: #{type}: #{message}" }
         log_file = instance.log_file
         # log error for further analysis (file must exist to activate)
         return if log_file.nil? || !File.exist?(log_file)

data/lib/aspera/rest_list.rb

@@ -17,8 +17,8 @@ module Aspera
     # @param query [Hash] Optional additional search query parameters
     # @return [Hash] A single entity matching the search, or an exception if not found or multiple found
     def lookup_with_q(entity, value:, field: 'name', query: {})
-      Aspera.assert_type(query, Hash){'query'}
-      Aspera.assert_type(field, String){'field'}
+      Aspera.assert_type(query, Hash) { 'query' }
+      Aspera.assert_type(field, String) { 'field' }
       # returns entities matching the query (it matches against several fields in case insensitive way)
       # We don't do paging, as anyway, we look for only one match
       matching_items = read(entity, query.merge({'q' => value}))
@@ -32,11 +32,11 @@ module Aspera
       else
         # multiple case insensitive partial matches, try case insensitive full match
         # (anyway AoC does not allow creation of 2 entities with same case insensitive field value)
-        value_matches = matching_items.select{ |i| i[field].casecmp?(value)}
+        value_matches = matching_items.select { |i| i[field].casecmp?(value) }
         case value_matches.length
         when 1 then return value_matches.first
-        when 0 then raise Error, "#{entity}: Multiple case insensitive partial match for: \"#{value}\" in #{matching_items.map{ |i| i[field]}.join(', ')} but no case insensitive full match. Please be more specific or give exact #{field}."
-        else raise Error, "Two entities cannot have the same case insensitive #{field}: #{value_matches.map{ |i| i[field]}}"
+        when 0 then raise Error, "#{entity}: Multiple case insensitive partial match for: \"#{value}\" in #{matching_items.map { |i| i[field] }.join(', ')} but no case insensitive full match. Please be more specific or give exact #{field}."
+        else raise Error, "Two entities cannot have the same case insensitive #{field}: #{value_matches.map { |i| i[field] }}"
         end
       end
     end
@@ -60,7 +60,8 @@ module Aspera
       Aspera.assert_type(entity, String)
       Aspera.assert_type(items_key, String)
       Aspera.assert_type(query, Hash)
-      Log.log.debug{"list_entities t=#{entity} k=#{items_key} q=#{query}"}
+      Log.log.debug { "list_entities t=#{entity} k=#{items_key}" }
+      Log.dump(:query, query)
       result = []
       offset = 0
       max_items = query.delete(MAX_ITEMS)
@@ -104,10 +105,10 @@ module Aspera
     # @param query     [Hash]   Additional query parameters (Default: `:default`)
     def lookup_entity_by_field(entity:, value:, field: 'name', items_key: nil, query: :default)
       if query.eql?(:default)
-        Aspera.assert_values(field, ['name']){'Default query field'}
+        Aspera.assert_values(field, ['name']) { 'Default query field' }
         query = {'q'=> value}
       end
-      lookup_entity_generic(entity: entity, field: field, value: value){list_entities_limit_offset_total_count(entity: entity, items_key: items_key, query: query).first}
+      lookup_entity_generic(entity: entity, field: field, value: value) { list_entities_limit_offset_total_count(entity: entity, items_key: items_key, query: query).first }
     end
 
     # Lookup entity by field and value.
@@ -123,7 +124,7 @@ module Aspera
       Aspera.assert(block_given?, 'block required for lookup_entity_generic')
       found = yield
       Aspera.assert_array_all(found, Hash)
-      found = found.select{ |i| i[field].eql?(value)}
+      found = found.select { |i| i[field].eql?(value) }
       return found.first if found.length.eql?(1)
       raise Cli::BadIdentifier.new(entity, value, field: field, count: found.length)
     end